2024-11-20 17:47:46,565 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 17:47:46,589 main DEBUG Took 0.011894 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 17:47:46,590 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 17:47:46,590 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 17:47:46,591 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 17:47:46,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,602 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 17:47:46,619 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,621 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,623 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,624 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,625 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,627 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,628 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,629 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,630 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,631 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,632 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,633 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:47:46,634 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,635 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 17:47:46,637 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:47:46,639 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 17:47:46,642 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 17:47:46,642 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 17:47:46,644 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 17:47:46,644 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 17:47:46,654 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 17:47:46,658 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 17:47:46,660 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 17:47:46,660 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 17:47:46,661 main DEBUG createAppenders(={Console}) 2024-11-20 17:47:46,662 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 17:47:46,662 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 17:47:46,663 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 17:47:46,663 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 17:47:46,664 main DEBUG OutputStream closed 2024-11-20 17:47:46,664 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 17:47:46,664 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 17:47:46,665 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 17:47:46,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 17:47:46,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 17:47:46,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 17:47:46,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 17:47:46,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 17:47:46,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 17:47:46,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 17:47:46,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 17:47:46,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 17:47:46,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 17:47:46,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 17:47:46,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 17:47:46,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 17:47:46,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 17:47:46,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 17:47:46,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 17:47:46,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 17:47:46,763 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 17:47:46,766 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 17:47:46,767 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 17:47:46,767 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 17:47:46,769 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T17:47:47,071 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249 2024-11-20 17:47:47,074 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 17:47:47,075 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T17:47:47,087 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-20T17:47:47,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T17:47:47,143 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b, deleteOnExit=true 2024-11-20T17:47:47,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T17:47:47,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/test.cache.data in system properties and HBase conf 2024-11-20T17:47:47,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T17:47:47,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/hadoop.log.dir in system properties and HBase conf 2024-11-20T17:47:47,149 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T17:47:47,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T17:47:47,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T17:47:47,261 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T17:47:47,377 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:47:47,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T17:47:47,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T17:47:47,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T17:47:47,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T17:47:47,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T17:47:47,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T17:47:47,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T17:47:47,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T17:47:47,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T17:47:47,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/nfs.dump.dir in system properties and HBase conf 2024-11-20T17:47:47,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/java.io.tmpdir in system properties and HBase conf 2024-11-20T17:47:47,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T17:47:47,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T17:47:47,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T17:47:48,938 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T17:47:49,012 INFO [Time-limited test {}] log.Log(170): Logging initialized @3196ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T17:47:49,092 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:47:49,160 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:47:49,187 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:47:49,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:47:49,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:47:49,202 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:47:49,205 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:47:49,207 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:47:49,391 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/java.io.tmpdir/jetty-localhost-41223-hadoop-hdfs-3_4_1-tests_jar-_-any-8592308062146394511/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T17:47:49,397 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:41223} 2024-11-20T17:47:49,397 INFO [Time-limited test {}] server.Server(415): Started @3581ms 2024-11-20T17:47:49,931 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:47:49,938 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:47:49,939 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:47:49,939 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:47:49,940 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:47:49,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:47:49,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:47:50,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/java.io.tmpdir/jetty-localhost-35335-hadoop-hdfs-3_4_1-tests_jar-_-any-8533677480003703999/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T17:47:50,040 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:35335} 2024-11-20T17:47:50,041 INFO [Time-limited test {}] server.Server(415): Started @4225ms 2024-11-20T17:47:50,088 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T17:47:51,075 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/data/data2/current/BP-1760351784-172.17.0.2-1732124868028/current, will proceed with Du for space computation calculation, 2024-11-20T17:47:51,075 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/data/data1/current/BP-1760351784-172.17.0.2-1732124868028/current, will proceed with Du for space computation calculation, 2024-11-20T17:47:51,104 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T17:47:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf941e77b262ff92d with lease ID 0x5527ac4854ab7b50: Processing first storage report for DS-10346f17-f2d7-4d54-ace5-6325364c0071 from datanode DatanodeRegistration(127.0.0.1:36597, datanodeUuid=9dae55b9-7ff8-4bc5-9284-abf32dffc5e9, infoPort=45203, infoSecurePort=0, ipcPort=32919, storageInfo=lv=-57;cid=testClusterID;nsid=714103592;c=1732124868028) 2024-11-20T17:47:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf941e77b262ff92d with lease ID 0x5527ac4854ab7b50: from storage DS-10346f17-f2d7-4d54-ace5-6325364c0071 node DatanodeRegistration(127.0.0.1:36597, datanodeUuid=9dae55b9-7ff8-4bc5-9284-abf32dffc5e9, infoPort=45203, infoSecurePort=0, ipcPort=32919, storageInfo=lv=-57;cid=testClusterID;nsid=714103592;c=1732124868028), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T17:47:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf941e77b262ff92d with lease ID 0x5527ac4854ab7b50: Processing first storage report for DS-947641db-127a-4863-80f7-199be4539d48 from datanode DatanodeRegistration(127.0.0.1:36597, datanodeUuid=9dae55b9-7ff8-4bc5-9284-abf32dffc5e9, infoPort=45203, infoSecurePort=0, ipcPort=32919, storageInfo=lv=-57;cid=testClusterID;nsid=714103592;c=1732124868028) 2024-11-20T17:47:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf941e77b262ff92d with lease ID 0x5527ac4854ab7b50: from storage DS-947641db-127a-4863-80f7-199be4539d48 node DatanodeRegistration(127.0.0.1:36597, datanodeUuid=9dae55b9-7ff8-4bc5-9284-abf32dffc5e9, infoPort=45203, infoSecurePort=0, ipcPort=32919, storageInfo=lv=-57;cid=testClusterID;nsid=714103592;c=1732124868028), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T17:47:51,183 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249 2024-11-20T17:47:51,262 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/zookeeper_0, clientPort=57706, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T17:47:51,272 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57706 2024-11-20T17:47:51,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:51,290 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:51,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741825_1001 (size=7) 2024-11-20T17:47:51,944 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 with version=8 2024-11-20T17:47:51,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/hbase-staging 2024-11-20T17:47:52,047 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T17:47:52,306 INFO [Time-limited test {}] client.ConnectionUtils(128): master/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:52,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:52,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:52,323 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:52,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:52,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:52,455 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T17:47:52,508 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T17:47:52,516 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T17:47:52,520 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:52,540 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21380 (auto-detected) 2024-11-20T17:47:52,541 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T17:47:52,557 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39775 2024-11-20T17:47:52,577 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39775 connecting to ZooKeeper ensemble=127.0.0.1:57706 2024-11-20T17:47:52,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397750x0, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:52,706 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39775-0x101595c05f50000 connected 2024-11-20T17:47:52,786 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:52,789 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:52,802 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:52,805 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7, hbase.cluster.distributed=false 2024-11-20T17:47:52,827 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:52,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39775 2024-11-20T17:47:52,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39775 2024-11-20T17:47:52,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39775 2024-11-20T17:47:52,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39775 2024-11-20T17:47:52,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39775 2024-11-20T17:47:52,923 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=45 2024-11-20T17:47:52,924 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:52,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:52,925 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:47:52,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:47:52,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:47:52,927 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:47:52,930 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:47:52,931 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44957 2024-11-20T17:47:52,932 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44957 connecting to ZooKeeper ensemble=127.0.0.1:57706 2024-11-20T17:47:52,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:52,936 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:52,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449570x0, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:47:52,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:449570x0, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:47:52,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44957-0x101595c05f50001 connected 2024-11-20T17:47:52,957 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:47:52,964 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:47:52,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:47:52,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:47:52,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44957 2024-11-20T17:47:52,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44957 2024-11-20T17:47:52,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44957 2024-11-20T17:47:52,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44957 2024-11-20T17:47:52,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44957 2024-11-20T17:47:52,989 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;613581fcdc40:39775 2024-11-20T17:47:52,989 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/613581fcdc40,39775,1732124872132 2024-11-20T17:47:53,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:53,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:53,007 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/613581fcdc40,39775,1732124872132 2024-11-20T17:47:53,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:47:53,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:53,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:53,037 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T17:47:53,039 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/613581fcdc40,39775,1732124872132 from backup master directory 2024-11-20T17:47:53,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:53,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/613581fcdc40,39775,1732124872132 2024-11-20T17:47:53,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:47:53,048 WARN [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:53,048 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=613581fcdc40,39775,1732124872132 2024-11-20T17:47:53,050 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T17:47:53,051 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T17:47:53,105 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/hbase.id] with ID: 1af8210c-6bb5-4317-b742-986938d243d3 2024-11-20T17:47:53,105 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/.tmp/hbase.id 2024-11-20T17:47:53,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741826_1002 (size=42) 2024-11-20T17:47:53,517 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/.tmp/hbase.id]:[hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/hbase.id] 2024-11-20T17:47:53,562 INFO [master/613581fcdc40:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:47:53,566 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T17:47:53,586 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-20T17:47:53,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:53,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:53,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741827_1003 (size=196) 2024-11-20T17:47:54,076 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:47:54,079 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T17:47:54,095 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:54,099 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741828_1004 (size=1189) 2024-11-20T17:47:54,151 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store 2024-11-20T17:47:54,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741829_1005 (size=34) 2024-11-20T17:47:54,578 INFO [master/613581fcdc40:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T17:47:54,582 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:54,583 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:47:54,583 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:54,584 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:54,585 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:47:54,586 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:54,586 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:47:54,587 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732124874583Disabling compacts and flushes for region at 1732124874583Disabling writes for close at 1732124874586 (+3 ms)Writing region close event to WAL at 1732124874586Closed at 1732124874586 2024-11-20T17:47:54,590 WARN [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/.initializing 2024-11-20T17:47:54,590 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/WALs/613581fcdc40,39775,1732124872132 2024-11-20T17:47:54,600 INFO [master/613581fcdc40:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:54,616 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C39775%2C1732124872132, suffix=, logDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/WALs/613581fcdc40,39775,1732124872132, archiveDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/oldWALs, maxLogs=10 2024-11-20T17:47:54,638 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/WALs/613581fcdc40,39775,1732124872132/613581fcdc40%2C39775%2C1732124872132.1732124874621, exclude list is [], retry=0 2024-11-20T17:47:54,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36597,DS-10346f17-f2d7-4d54-ace5-6325364c0071,DISK] 2024-11-20T17:47:54,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T17:47:54,687 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/WALs/613581fcdc40,39775,1732124872132/613581fcdc40%2C39775%2C1732124872132.1732124874621 2024-11-20T17:47:54,688 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45203:45203)] 2024-11-20T17:47:54,689 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:54,689 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:54,692 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,693 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T17:47:54,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:54,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:54,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T17:47:54,757 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:54,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:54,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T17:47:54,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:54,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:54,764 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T17:47:54,767 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:54,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:54,768 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,771 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,772 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,777 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,778 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,782 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:47:54,785 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:47:54,790 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:54,791 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65171227, jitterRate=-0.02887304127216339}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:47:54,797 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732124874704Initializing all the Stores at 1732124874706 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124874707 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124874707Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124874708 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124874708Cleaning up temporary data from old regions at 1732124874778 (+70 ms)Region opened successfully at 1732124874797 (+19 ms) 2024-11-20T17:47:54,798 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T17:47:54,830 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd680fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:54,858 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T17:47:54,868 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T17:47:54,868 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T17:47:54,871 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:47:54,872 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T17:47:54,877 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T17:47:54,877 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T17:47:54,906 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T17:47:54,914 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T17:47:54,962 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T17:47:54,965 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T17:47:54,967 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T17:47:54,973 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T17:47:54,976 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T17:47:54,980 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T17:47:55,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T17:47:55,051 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T17:47:55,131 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T17:47:55,153 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T17:47:55,173 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T17:47:55,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:55,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:47:55,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:55,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:55,188 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=613581fcdc40,39775,1732124872132, sessionid=0x101595c05f50000, setting cluster-up flag (Was=false) 2024-11-20T17:47:55,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:55,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:55,247 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T17:47:55,250 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,39775,1732124872132 2024-11-20T17:47:55,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:55,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:55,310 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T17:47:55,314 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,39775,1732124872132 2024-11-20T17:47:55,322 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T17:47:55,380 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(746): ClusterId : 1af8210c-6bb5-4317-b742-986938d243d3 2024-11-20T17:47:55,383 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:47:55,387 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:55,396 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:47:55,396 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:47:55,397 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T17:47:55,402 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T17:47:55,406 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:47:55,406 DEBUG [RS:0;613581fcdc40:44957 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562a40ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:47:55,408 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 613581fcdc40,39775,1732124872132 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T17:47:55,417 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:55,418 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:55,418 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:55,419 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:47:55,419 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/613581fcdc40:0, corePoolSize=10, maxPoolSize=10 2024-11-20T17:47:55,419 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,419 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:55,420 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,421 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;613581fcdc40:44957 2024-11-20T17:47:55,424 INFO [RS:0;613581fcdc40:44957 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:47:55,424 INFO [RS:0;613581fcdc40:44957 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:47:55,424 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:47:55,424 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732124905424 2024-11-20T17:47:55,425 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:55,425 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T17:47:55,426 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T17:47:55,427 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,39775,1732124872132 with port=44957, startcode=1732124872893 2024-11-20T17:47:55,427 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T17:47:55,431 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T17:47:55,432 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T17:47:55,432 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T17:47:55,432 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T17:47:55,432 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:55,433 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T17:47:55,433 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,437 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T17:47:55,438 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T17:47:55,438 DEBUG [RS:0;613581fcdc40:44957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:47:55,438 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T17:47:55,442 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T17:47:55,442 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T17:47:55,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741831_1007 (size=1321) 2024-11-20T17:47:55,448 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124875444,5,FailOnTimeoutGroup] 2024-11-20T17:47:55,449 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124875449,5,FailOnTimeoutGroup] 2024-11-20T17:47:55,449 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,449 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T17:47:55,450 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,451 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47067, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:47:55,479 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39775 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,44957,1732124872893 2024-11-20T17:47:55,481 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39775 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,44957,1732124872893 2024-11-20T17:47:55,495 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:47:55,496 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38835 2024-11-20T17:47:55,496 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:47:55,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:47:55,576 DEBUG [RS:0;613581fcdc40:44957 {}] zookeeper.ZKUtil(111): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,44957,1732124872893 2024-11-20T17:47:55,576 WARN [RS:0;613581fcdc40:44957 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:47:55,577 INFO [RS:0;613581fcdc40:44957 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:55,577 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893 2024-11-20T17:47:55,581 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,44957,1732124872893] 2024-11-20T17:47:55,603 INFO [RS:0;613581fcdc40:44957 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:47:55,619 INFO [RS:0;613581fcdc40:44957 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:47:55,624 INFO [RS:0;613581fcdc40:44957 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:47:55,624 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,625 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:47:55,630 INFO [RS:0;613581fcdc40:44957 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:47:55,631 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,631 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,631 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,631 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,632 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,632 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,632 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:47:55,632 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,632 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,632 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,633 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,633 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,633 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:47:55,633 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:55,633 DEBUG [RS:0;613581fcdc40:44957 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:47:55,634 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,634 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,634 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,634 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,635 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,635 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,44957,1732124872893-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:55,655 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:47:55,656 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,44957,1732124872893-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,657 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,657 INFO [RS:0;613581fcdc40:44957 {}] regionserver.Replication(171): 613581fcdc40,44957,1732124872893 started 2024-11-20T17:47:55,675 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:55,676 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,44957,1732124872893, RpcServer on 613581fcdc40/172.17.0.2:44957, sessionid=0x101595c05f50001 2024-11-20T17:47:55,677 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:47:55,677 DEBUG [RS:0;613581fcdc40:44957 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,44957,1732124872893 2024-11-20T17:47:55,677 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,44957,1732124872893' 2024-11-20T17:47:55,678 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:47:55,679 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:47:55,679 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:47:55,680 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:47:55,680 DEBUG [RS:0;613581fcdc40:44957 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,44957,1732124872893 2024-11-20T17:47:55,680 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,44957,1732124872893' 2024-11-20T17:47:55,680 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:47:55,681 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:47:55,682 DEBUG [RS:0;613581fcdc40:44957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:47:55,682 INFO [RS:0;613581fcdc40:44957 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:47:55,682 INFO [RS:0;613581fcdc40:44957 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:47:55,786 INFO [RS:0;613581fcdc40:44957 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:47:55,790 INFO [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C44957%2C1732124872893, suffix=, logDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893, archiveDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/oldWALs, maxLogs=32 2024-11-20T17:47:55,804 DEBUG [RS:0;613581fcdc40:44957 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893/613581fcdc40%2C44957%2C1732124872893.1732124875792, exclude list is [], retry=0 2024-11-20T17:47:55,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36597,DS-10346f17-f2d7-4d54-ace5-6325364c0071,DISK] 2024-11-20T17:47:55,812 INFO [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893/613581fcdc40%2C44957%2C1732124872893.1732124875792 2024-11-20T17:47:55,812 DEBUG [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45203:45203)] 2024-11-20T17:47:55,848 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T17:47:55,849 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:47:55,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741833_1009 (size=32) 2024-11-20T17:47:56,263 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:56,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T17:47:56,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:56,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T17:47:56,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:56,274 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T17:47:56,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:56,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T17:47:56,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:56,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:56,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740 2024-11-20T17:47:56,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740 2024-11-20T17:47:56,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:56,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:56,287 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:47:56,289 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:56,293 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:56,294 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72510204, jitterRate=0.0804862380027771}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:47:56,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732124876264Initializing all the Stores at 1732124876266 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124876266Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124876266Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124876266Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124876266Cleaning up temporary data from old regions at 1732124876286 (+20 ms)Region opened successfully at 1732124876297 (+11 ms) 2024-11-20T17:47:56,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:47:56,297 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:47:56,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:47:56,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:47:56,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:47:56,299 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:47:56,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732124876297Disabling compacts and flushes for region at 1732124876297Disabling writes for close at 1732124876298 (+1 ms)Writing region close event to WAL at 1732124876299 (+1 ms)Closed at 1732124876299 2024-11-20T17:47:56,303 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:56,303 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T17:47:56,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:47:56,317 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:56,320 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T17:47:56,473 DEBUG [613581fcdc40:39775 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T17:47:56,484 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:47:56,490 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,44957,1732124872893, state=OPENING 2024-11-20T17:47:56,562 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T17:47:56,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:56,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:47:56,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:56,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:56,576 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:47:56,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:47:56,752 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T17:47:56,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45701, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T17:47:56,767 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T17:47:56,768 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:47:56,768 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T17:47:56,772 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C44957%2C1732124872893.meta, suffix=.meta, logDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893, archiveDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/oldWALs, maxLogs=32 2024-11-20T17:47:56,786 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893/613581fcdc40%2C44957%2C1732124872893.meta.1732124876774.meta, exclude list is [], retry=0 2024-11-20T17:47:56,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36597,DS-10346f17-f2d7-4d54-ace5-6325364c0071,DISK] 2024-11-20T17:47:56,793 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/WALs/613581fcdc40,44957,1732124872893/613581fcdc40%2C44957%2C1732124872893.meta.1732124876774.meta 2024-11-20T17:47:56,794 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45203:45203)] 2024-11-20T17:47:56,794 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:56,796 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T17:47:56,799 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T17:47:56,803 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T17:47:56,807 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T17:47:56,807 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:56,808 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T17:47:56,808 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T17:47:56,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T17:47:56,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:47:56,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T17:47:56,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:47:56,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T17:47:56,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:47:56,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T17:47:56,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:47:56,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:56,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:47:56,820 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:47:56,822 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740 2024-11-20T17:47:56,825 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740 2024-11-20T17:47:56,827 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:47:56,828 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:47:56,829 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:47:56,832 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:47:56,834 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61395618, jitterRate=-0.08513399958610535}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:47:56,834 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T17:47:56,836 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732124876808Writing region info on filesystem at 1732124876809 (+1 ms)Initializing all the Stores at 1732124876810 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124876811 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124876811Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124876811Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732124876811Cleaning up temporary data from old regions at 1732124876828 (+17 ms)Running coprocessor post-open hooks at 1732124876834 (+6 ms)Region opened successfully at 1732124876836 (+2 ms) 2024-11-20T17:47:56,843 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732124876744 2024-11-20T17:47:56,854 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T17:47:56,854 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T17:47:56,856 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:47:56,859 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,44957,1732124872893, state=OPEN 2024-11-20T17:47:56,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:56,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T17:47:56,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:56,894 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T17:47:56,895 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:56,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T17:47:56,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,44957,1732124872893 in 317 msec 2024-11-20T17:47:56,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T17:47:56,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 593 msec 2024-11-20T17:47:56,909 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:47:56,909 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T17:47:56,928 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:56,929 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:56,947 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:56,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47801, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:56,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6180 sec 2024-11-20T17:47:56,969 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732124876969, completionTime=-1 2024-11-20T17:47:56,971 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T17:47:56,971 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T17:47:56,997 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-20T17:47:56,997 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732124936997 2024-11-20T17:47:56,997 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732124996997 2024-11-20T17:47:56,997 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-20T17:47:57,000 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,39775,1732124872132-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,001 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,39775,1732124872132-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,001 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,39775,1732124872132-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,003 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-613581fcdc40:39775, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,003 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,004 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,010 DEBUG [master/613581fcdc40:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T17:47:57,034 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.986sec 2024-11-20T17:47:57,036 INFO [master/613581fcdc40:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T17:47:57,037 INFO [master/613581fcdc40:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T17:47:57,038 INFO [master/613581fcdc40:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T17:47:57,039 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T17:47:57,039 INFO [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T17:47:57,040 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,39775,1732124872132-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:47:57,040 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,39775,1732124872132-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T17:47:57,049 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T17:47:57,050 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T17:47:57,051 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,39775,1732124872132-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:47:57,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:57,091 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T17:47:57,091 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T17:47:57,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:57,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:57,106 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:57,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:57,119 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:57,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:57,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:57,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:57,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:57,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:57,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=613581fcdc40,39775,1732124872132 2024-11-20T17:47:57,169 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=392, ProcessCount=11, AvailableMemoryMB=9209 2024-11-20T17:47:57,177 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:47:57,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:47:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-20T17:47:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:47:57,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:47:57,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:47:57,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:47:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T17:47:57,221 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:47:57,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T17:47:57,224 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:57,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:47:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:57,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741835_1011 (size=959) 2024-11-20T17:47:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:57,664 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:47:57,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741836_1012 (size=53) 2024-11-20T17:47:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:58,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:58,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f6561bfd37132223c1b60b9e3a6231d8, disabling compactions & flushes 2024-11-20T17:47:58,079 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. after waiting 0 ms 2024-11-20T17:47:58,080 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,080 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f6561bfd37132223c1b60b9e3a6231d8: Waiting for close lock at 1732124878079Disabling compacts and flushes for region at 1732124878079Disabling writes for close at 1732124878080 (+1 ms)Writing region close event to WAL at 1732124878080Closed at 1732124878080 2024-11-20T17:47:58,085 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:47:58,090 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732124878085"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732124878085"}]},"ts":"1732124878085"} 2024-11-20T17:47:58,095 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:47:58,097 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:47:58,099 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124878097"}]},"ts":"1732124878097"} 2024-11-20T17:47:58,103 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T17:47:58,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, ASSIGN}] 2024-11-20T17:47:58,108 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, ASSIGN 2024-11-20T17:47:58,109 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, ASSIGN; state=OFFLINE, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=false 2024-11-20T17:47:58,261 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f6561bfd37132223c1b60b9e3a6231d8, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:47:58,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, ASSIGN because future has completed 2024-11-20T17:47:58,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:47:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:58,434 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,435 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:47:58,435 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,436 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:47:58,436 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,436 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,439 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,442 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6561bfd37132223c1b60b9e3a6231d8 columnFamilyName A 2024-11-20T17:47:58,442 DEBUG [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:58,443 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] regionserver.HStore(327): Store=f6561bfd37132223c1b60b9e3a6231d8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:58,444 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,446 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6561bfd37132223c1b60b9e3a6231d8 columnFamilyName B 2024-11-20T17:47:58,446 DEBUG [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:58,447 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] regionserver.HStore(327): Store=f6561bfd37132223c1b60b9e3a6231d8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:58,448 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,450 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6561bfd37132223c1b60b9e3a6231d8 columnFamilyName C 2024-11-20T17:47:58,450 DEBUG [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:47:58,451 INFO [StoreOpener-f6561bfd37132223c1b60b9e3a6231d8-1 {}] regionserver.HStore(327): Store=f6561bfd37132223c1b60b9e3a6231d8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:47:58,452 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,453 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,454 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,456 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,456 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,457 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:47:58,459 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,463 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:47:58,464 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f6561bfd37132223c1b60b9e3a6231d8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59292858, jitterRate=-0.11646756529808044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:47:58,464 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:58,465 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f6561bfd37132223c1b60b9e3a6231d8: Running coprocessor pre-open hook at 1732124878436Writing region info on filesystem at 1732124878436Initializing all the Stores at 1732124878438 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124878439 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124878439Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124878439Cleaning up temporary data from old regions at 1732124878456 (+17 ms)Running coprocessor post-open hooks at 1732124878464 (+8 ms)Region opened successfully at 1732124878465 (+1 ms) 2024-11-20T17:47:58,467 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., pid=6, masterSystemTime=1732124878424 2024-11-20T17:47:58,470 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,470 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:58,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f6561bfd37132223c1b60b9e3a6231d8, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:47:58,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:47:58,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T17:47:58,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 in 209 msec 2024-11-20T17:47:58,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T17:47:58,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, ASSIGN in 379 msec 2024-11-20T17:47:58,490 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:47:58,490 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124878490"}]},"ts":"1732124878490"} 2024-11-20T17:47:58,493 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T17:47:58,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:47:58,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2820 sec 2024-11-20T17:47:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T17:47:59,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:47:59,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,401 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,407 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:47:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:47:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a6cc6c1 2024-11-20T17:47:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:47:59,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:47:59,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,426 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,433 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,442 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,457 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,460 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dae11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,463 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,465 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4e141f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,468 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,471 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57865cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:47:59,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:47:59,474 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:47:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:47:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:47:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463070d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:47:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:47:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:47:59,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:47:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5c6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:47:59,481 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,482 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,482 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,482 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,482 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,483 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,483 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,486 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,486 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:47:59,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,489 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:47:59,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:47:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T17:47:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:47:59,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:47:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:47:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,502 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:47:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T17:47:59,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:47:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:47:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:47:59,573 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:47:59,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:47:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T17:47:59,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:47:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:59,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:47:59,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:59,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:59,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:59,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/a71b5caa62254dab9ff7b71e505a5c28 is 50, key is test_row_0/A:col10/1732124879569/Put/seqid=0 2024-11-20T17:47:59,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124939755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124939752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124939756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741837_1013 (size=14341) 2024-11-20T17:47:59,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124939777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/a71b5caa62254dab9ff7b71e505a5c28 2024-11-20T17:47:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124939778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T17:47:59,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:47:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:47:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:47:59,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:47:59,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124939919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124939919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124939920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124939920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:47:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124939920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:47:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:47:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:47:59,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/931da03cc60141a5b1288f78b5b8054c is 50, key is test_row_0/B:col10/1732124879569/Put/seqid=0 2024-11-20T17:47:59,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741838_1014 (size=12001) 2024-11-20T17:48:00,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:48:00,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T17:48:00,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124940139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124940140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124940140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124940140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124940141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:48:00,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:48:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/931da03cc60141a5b1288f78b5b8054c 2024-11-20T17:48:00,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2d1d57042b1f4c78a1357bf90cca29e0 is 50, key is test_row_0/C:col10/1732124879569/Put/seqid=0 2024-11-20T17:48:00,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124940448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124940449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124940458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124940459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124940460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:00,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:00,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741839_1015 (size=12001) 2024-11-20T17:48:00,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2d1d57042b1f4c78a1357bf90cca29e0 2024-11-20T17:48:00,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:48:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:00,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/a71b5caa62254dab9ff7b71e505a5c28 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a71b5caa62254dab9ff7b71e505a5c28 2024-11-20T17:48:00,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:00,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a71b5caa62254dab9ff7b71e505a5c28, entries=200, sequenceid=13, filesize=14.0 K 2024-11-20T17:48:00,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/931da03cc60141a5b1288f78b5b8054c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/931da03cc60141a5b1288f78b5b8054c 2024-11-20T17:48:00,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/931da03cc60141a5b1288f78b5b8054c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:48:00,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2d1d57042b1f4c78a1357bf90cca29e0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2d1d57042b1f4c78a1357bf90cca29e0 2024-11-20T17:48:00,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2d1d57042b1f4c78a1357bf90cca29e0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:48:00,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 1019ms, sequenceid=13, compaction requested=false 2024-11-20T17:48:00,600 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T17:48:00,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T17:48:00,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T17:48:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,685 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:48:00,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/872af24220bd4aabb6b633efe43032d0 is 50, key is test_row_0/A:col10/1732124879768/Put/seqid=0 2024-11-20T17:48:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741840_1016 (size=12001) 2024-11-20T17:48:00,752 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/872af24220bd4aabb6b633efe43032d0 2024-11-20T17:48:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/dc967efb93a34519896e52c8c65e83a6 is 50, key is test_row_0/B:col10/1732124879768/Put/seqid=0 2024-11-20T17:48:00,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741841_1017 (size=12001) 2024-11-20T17:48:00,842 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/dc967efb93a34519896e52c8c65e83a6 2024-11-20T17:48:00,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a94a16d5f2a9477e8e5767c6b428aa07 is 50, key is test_row_0/C:col10/1732124879768/Put/seqid=0 2024-11-20T17:48:00,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741842_1018 (size=12001) 2024-11-20T17:48:00,898 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a94a16d5f2a9477e8e5767c6b428aa07 2024-11-20T17:48:00,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/872af24220bd4aabb6b633efe43032d0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/872af24220bd4aabb6b633efe43032d0 2024-11-20T17:48:00,940 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/872af24220bd4aabb6b633efe43032d0, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T17:48:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/dc967efb93a34519896e52c8c65e83a6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/dc967efb93a34519896e52c8c65e83a6 2024-11-20T17:48:00,962 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/dc967efb93a34519896e52c8c65e83a6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T17:48:00,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a94a16d5f2a9477e8e5767c6b428aa07 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a94a16d5f2a9477e8e5767c6b428aa07 2024-11-20T17:48:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:00,990 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a94a16d5f2a9477e8e5767c6b428aa07, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T17:48:00,991 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=40.25 KB/41220 for f6561bfd37132223c1b60b9e3a6231d8 in 306ms, sequenceid=38, compaction requested=false 2024-11-20T17:48:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:00,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T17:48:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T17:48:01,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T17:48:01,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4930 sec 2024-11-20T17:48:01,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5180 sec 2024-11-20T17:48:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:01,015 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:01,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:01,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/77e3408d65604dfdb8e7b2685f08eed2 is 50, key is test_row_0/A:col10/1732124881012/Put/seqid=0 2024-11-20T17:48:01,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741843_1019 (size=19021) 2024-11-20T17:48:01,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/77e3408d65604dfdb8e7b2685f08eed2 2024-11-20T17:48:01,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c50eb8ddd28c4af8892b2b2901ed3fcb is 50, key is test_row_0/B:col10/1732124881012/Put/seqid=0 2024-11-20T17:48:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741844_1020 (size=12001) 2024-11-20T17:48:01,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c50eb8ddd28c4af8892b2b2901ed3fcb 2024-11-20T17:48:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124941112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124941113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124941119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124941119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/c54d0658826d4bb99b40e004ae10d822 is 50, key is test_row_0/C:col10/1732124881012/Put/seqid=0 2024-11-20T17:48:01,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124941123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741845_1021 (size=12001) 2024-11-20T17:48:01,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/c54d0658826d4bb99b40e004ae10d822 2024-11-20T17:48:01,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/77e3408d65604dfdb8e7b2685f08eed2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/77e3408d65604dfdb8e7b2685f08eed2 2024-11-20T17:48:01,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/77e3408d65604dfdb8e7b2685f08eed2, entries=300, sequenceid=49, filesize=18.6 K 2024-11-20T17:48:01,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c50eb8ddd28c4af8892b2b2901ed3fcb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c50eb8ddd28c4af8892b2b2901ed3fcb 2024-11-20T17:48:01,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c50eb8ddd28c4af8892b2b2901ed3fcb, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T17:48:01,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/c54d0658826d4bb99b40e004ae10d822 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c54d0658826d4bb99b40e004ae10d822 2024-11-20T17:48:01,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c54d0658826d4bb99b40e004ae10d822, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T17:48:01,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 216ms, sequenceid=49, compaction requested=true 2024-11-20T17:48:01,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:01,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:01,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:01,237 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:01,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:01,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:01,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:01,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:48:01,239 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:01,242 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:01,244 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:01,244 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:01,244 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:01,245 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:01,245 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:01,245 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/931da03cc60141a5b1288f78b5b8054c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/dc967efb93a34519896e52c8c65e83a6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c50eb8ddd28c4af8892b2b2901ed3fcb] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=35.2 K 2024-11-20T17:48:01,245 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a71b5caa62254dab9ff7b71e505a5c28, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/872af24220bd4aabb6b633efe43032d0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/77e3408d65604dfdb8e7b2685f08eed2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=44.3 K 2024-11-20T17:48:01,247 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 931da03cc60141a5b1288f78b5b8054c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732124879550 2024-11-20T17:48:01,247 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71b5caa62254dab9ff7b71e505a5c28, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732124879529 2024-11-20T17:48:01,248 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting dc967efb93a34519896e52c8c65e83a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732124879735 2024-11-20T17:48:01,250 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c50eb8ddd28c4af8892b2b2901ed3fcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732124880982 2024-11-20T17:48:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:01,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T17:48:01,252 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872af24220bd4aabb6b633efe43032d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732124879735 2024-11-20T17:48:01,254 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77e3408d65604dfdb8e7b2685f08eed2, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732124880982 2024-11-20T17:48:01,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124941262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/9ab8a2f2b39147f5b0345dd9c0fefd06 is 50, key is test_row_0/A:col10/1732124881117/Put/seqid=0 2024-11-20T17:48:01,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124941273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124941273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124941277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124941277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,298 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:01,299 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/78c179f3a35e4798a834171b29e2e055 is 50, key is test_row_0/A:col10/1732124881012/Put/seqid=0 2024-11-20T17:48:01,301 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#11 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:01,302 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe is 50, key is test_row_0/B:col10/1732124881012/Put/seqid=0 2024-11-20T17:48:01,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741846_1022 (size=16681) 2024-11-20T17:48:01,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/9ab8a2f2b39147f5b0345dd9c0fefd06 2024-11-20T17:48:01,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741847_1023 (size=12104) 2024-11-20T17:48:01,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741848_1024 (size=12104) 2024-11-20T17:48:01,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5ba87428b4ba42f69218ce5cf3e08c69 is 50, key is test_row_0/B:col10/1732124881117/Put/seqid=0 2024-11-20T17:48:01,357 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/78c179f3a35e4798a834171b29e2e055 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/78c179f3a35e4798a834171b29e2e055 2024-11-20T17:48:01,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741849_1025 (size=12001) 2024-11-20T17:48:01,385 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 78c179f3a35e4798a834171b29e2e055(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:01,385 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:01,390 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124881233; duration=0sec 2024-11-20T17:48:01,390 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:01,391 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:01,391 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124941387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,397 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:01,397 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:01,397 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:01,397 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2d1d57042b1f4c78a1357bf90cca29e0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a94a16d5f2a9477e8e5767c6b428aa07, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c54d0658826d4bb99b40e004ae10d822] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=35.2 K 2024-11-20T17:48:01,399 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1d57042b1f4c78a1357bf90cca29e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732124879550 2024-11-20T17:48:01,400 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a94a16d5f2a9477e8e5767c6b428aa07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732124879735 2024-11-20T17:48:01,401 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c54d0658826d4bb99b40e004ae10d822, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732124880982 2024-11-20T17:48:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124941398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124941399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124941399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124941399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,424 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:01,425 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/44727a01ff6941c4831678ca864da11e is 50, key is test_row_0/C:col10/1732124881012/Put/seqid=0 2024-11-20T17:48:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741850_1026 (size=12104) 2024-11-20T17:48:01,465 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/44727a01ff6941c4831678ca864da11e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/44727a01ff6941c4831678ca864da11e 2024-11-20T17:48:01,482 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 44727a01ff6941c4831678ca864da11e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:01,482 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:01,482 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=13, startTime=1732124881237; duration=0sec 2024-11-20T17:48:01,482 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:01,482 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:01,599 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T17:48:01,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124941598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124941618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124941618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124941619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124941619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T17:48:01,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:01,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T17:48:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T17:48:01,672 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:01,674 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:01,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:01,756 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe 2024-11-20T17:48:01,774 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into ff65fa1bdcc94dd6a94ac3605a7d5bfe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:01,774 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:01,774 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124881237; duration=0sec 2024-11-20T17:48:01,774 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:01,774 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T17:48:01,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5ba87428b4ba42f69218ce5cf3e08c69 2024-11-20T17:48:01,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/349e46a2380b421f95a721084762bdd9 is 50, key is test_row_0/C:col10/1732124881117/Put/seqid=0 2024-11-20T17:48:01,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T17:48:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:01,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:01,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:01,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741851_1027 (size=12001) 2024-11-20T17:48:01,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/349e46a2380b421f95a721084762bdd9 2024-11-20T17:48:01,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/9ab8a2f2b39147f5b0345dd9c0fefd06 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/9ab8a2f2b39147f5b0345dd9c0fefd06 2024-11-20T17:48:01,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/9ab8a2f2b39147f5b0345dd9c0fefd06, entries=250, sequenceid=76, filesize=16.3 K 2024-11-20T17:48:01,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5ba87428b4ba42f69218ce5cf3e08c69 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5ba87428b4ba42f69218ce5cf3e08c69 2024-11-20T17:48:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124941908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124941937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124941938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124941938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124941938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:01,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5ba87428b4ba42f69218ce5cf3e08c69, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T17:48:01,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/349e46a2380b421f95a721084762bdd9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/349e46a2380b421f95a721084762bdd9 2024-11-20T17:48:01,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/349e46a2380b421f95a721084762bdd9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T17:48:01,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for f6561bfd37132223c1b60b9e3a6231d8 in 727ms, sequenceid=76, compaction requested=false 2024-11-20T17:48:01,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:01,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T17:48:01,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:01,987 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T17:48:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T17:48:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d213a36d10a24fb7b5e2cfa976c05874 is 50, key is test_row_0/A:col10/1732124881257/Put/seqid=0 2024-11-20T17:48:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741852_1028 (size=12001) 2024-11-20T17:48:02,023 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d213a36d10a24fb7b5e2cfa976c05874 2024-11-20T17:48:02,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/e55647bbb3f94d688a9520cfe13a231d is 50, key is test_row_0/B:col10/1732124881257/Put/seqid=0 2024-11-20T17:48:02,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741853_1029 (size=12001) 2024-11-20T17:48:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T17:48:02,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:48:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:02,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T17:48:02,506 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T17:48:02,508 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:48:02,509 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T17:48:02,509 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T17:48:02,509 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T17:48:02,510 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/e55647bbb3f94d688a9520cfe13a231d 2024-11-20T17:48:02,512 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T17:48:02,512 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T17:48:02,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124942499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124942512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124942513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124942516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124942516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2928cd17970f45e8938afd410f52a3dd is 50, key is test_row_0/C:col10/1732124881257/Put/seqid=0 2024-11-20T17:48:02,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741854_1030 (size=12001) 2024-11-20T17:48:02,579 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2928cd17970f45e8938afd410f52a3dd 2024-11-20T17:48:02,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d213a36d10a24fb7b5e2cfa976c05874 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d213a36d10a24fb7b5e2cfa976c05874 2024-11-20T17:48:02,615 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d213a36d10a24fb7b5e2cfa976c05874, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T17:48:02,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/e55647bbb3f94d688a9520cfe13a231d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e55647bbb3f94d688a9520cfe13a231d 2024-11-20T17:48:02,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124942628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124942628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,640 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e55647bbb3f94d688a9520cfe13a231d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T17:48:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2928cd17970f45e8938afd410f52a3dd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2928cd17970f45e8938afd410f52a3dd 2024-11-20T17:48:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124942637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124942637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124942638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,660 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2928cd17970f45e8938afd410f52a3dd, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T17:48:02,663 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for f6561bfd37132223c1b60b9e3a6231d8 in 675ms, sequenceid=89, compaction requested=true 2024-11-20T17:48:02,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:02,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:02,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T17:48:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T17:48:02,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T17:48:02,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-20T17:48:02,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0050 sec 2024-11-20T17:48:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T17:48:02,808 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:02,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T17:48:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T17:48:02,819 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:02,821 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:02,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:02,843 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:02,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T17:48:02,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b03651d7f8264e89a560438a8771de7e is 50, key is test_row_0/A:col10/1732124882512/Put/seqid=0 2024-11-20T17:48:02,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124942858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124942863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124942864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124942865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124942874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741855_1031 (size=14341) 2024-11-20T17:48:02,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b03651d7f8264e89a560438a8771de7e 2024-11-20T17:48:02,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/97ba47a7bb6049b39782ee27c8739c56 is 50, key is test_row_0/B:col10/1732124882512/Put/seqid=0 2024-11-20T17:48:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T17:48:02,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741856_1032 (size=12001) 2024-11-20T17:48:02,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/97ba47a7bb6049b39782ee27c8739c56 2024-11-20T17:48:02,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/5e806049c05648d18b22fc5e415709b4 is 50, key is test_row_0/C:col10/1732124882512/Put/seqid=0 2024-11-20T17:48:02,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:02,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124942979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124942979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124942979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:02,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124942988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741857_1033 (size=12001) 2024-11-20T17:48:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T17:48:03,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124943177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124943188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124943197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124943197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124943198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/5e806049c05648d18b22fc5e415709b4 2024-11-20T17:48:03,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b03651d7f8264e89a560438a8771de7e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b03651d7f8264e89a560438a8771de7e 2024-11-20T17:48:03,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b03651d7f8264e89a560438a8771de7e, entries=200, sequenceid=116, filesize=14.0 K 2024-11-20T17:48:03,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/97ba47a7bb6049b39782ee27c8739c56 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/97ba47a7bb6049b39782ee27c8739c56 2024-11-20T17:48:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T17:48:03,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:03,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/97ba47a7bb6049b39782ee27c8739c56, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T17:48:03,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/5e806049c05648d18b22fc5e415709b4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/5e806049c05648d18b22fc5e415709b4 2024-11-20T17:48:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/5e806049c05648d18b22fc5e415709b4, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T17:48:03,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for f6561bfd37132223c1b60b9e3a6231d8 in 623ms, sequenceid=116, compaction requested=true 2024-11-20T17:48:03,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:03,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:03,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:03,468 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:03,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:03,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:03,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:03,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:48:03,468 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:03,473 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:03,473 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:03,473 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,473 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/78c179f3a35e4798a834171b29e2e055, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/9ab8a2f2b39147f5b0345dd9c0fefd06, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d213a36d10a24fb7b5e2cfa976c05874, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b03651d7f8264e89a560438a8771de7e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=53.8 K 2024-11-20T17:48:03,474 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c179f3a35e4798a834171b29e2e055, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732124880982 2024-11-20T17:48:03,475 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:03,475 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab8a2f2b39147f5b0345dd9c0fefd06, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732124881102 2024-11-20T17:48:03,475 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:03,475 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,476 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5ba87428b4ba42f69218ce5cf3e08c69, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e55647bbb3f94d688a9520cfe13a231d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/97ba47a7bb6049b39782ee27c8739c56] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=47.0 K 2024-11-20T17:48:03,477 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d213a36d10a24fb7b5e2cfa976c05874, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732124881257 2024-11-20T17:48:03,477 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff65fa1bdcc94dd6a94ac3605a7d5bfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732124880982 2024-11-20T17:48:03,478 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b03651d7f8264e89a560438a8771de7e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732124882507 2024-11-20T17:48:03,478 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba87428b4ba42f69218ce5cf3e08c69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732124881117 2024-11-20T17:48:03,478 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e55647bbb3f94d688a9520cfe13a231d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732124881257 2024-11-20T17:48:03,480 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97ba47a7bb6049b39782ee27c8739c56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732124882507 2024-11-20T17:48:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:03,509 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:03,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:03,509 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#21 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:03,510 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d894c04c40a040a1b630867cbdd15c6b is 50, key is test_row_0/A:col10/1732124882512/Put/seqid=0 2024-11-20T17:48:03,531 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:03,532 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/15a8cb6c4a65487592c65f069bcb94d1 is 50, key is test_row_0/B:col10/1732124882512/Put/seqid=0 2024-11-20T17:48:03,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/869dd43873c7436e87dfd4ffeefc62ab is 50, key is test_row_0/A:col10/1732124882862/Put/seqid=0 2024-11-20T17:48:03,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741858_1034 (size=12241) 2024-11-20T17:48:03,566 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d894c04c40a040a1b630867cbdd15c6b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d894c04c40a040a1b630867cbdd15c6b 2024-11-20T17:48:03,577 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into d894c04c40a040a1b630867cbdd15c6b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:03,577 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:03,578 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=12, startTime=1732124883467; duration=0sec 2024-11-20T17:48:03,578 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:03,578 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:03,578 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:03,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741859_1035 (size=12241) 2024-11-20T17:48:03,584 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:03,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:03,585 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,585 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/44727a01ff6941c4831678ca864da11e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/349e46a2380b421f95a721084762bdd9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2928cd17970f45e8938afd410f52a3dd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/5e806049c05648d18b22fc5e415709b4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=47.0 K 2024-11-20T17:48:03,586 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 44727a01ff6941c4831678ca864da11e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732124880982 2024-11-20T17:48:03,590 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 349e46a2380b421f95a721084762bdd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732124881117 2024-11-20T17:48:03,600 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2928cd17970f45e8938afd410f52a3dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732124881257 2024-11-20T17:48:03,600 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/15a8cb6c4a65487592c65f069bcb94d1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/15a8cb6c4a65487592c65f069bcb94d1 2024-11-20T17:48:03,601 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e806049c05648d18b22fc5e415709b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732124882507 2024-11-20T17:48:03,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741860_1036 (size=14341) 2024-11-20T17:48:03,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/869dd43873c7436e87dfd4ffeefc62ab 2024-11-20T17:48:03,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124943608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,622 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into 15a8cb6c4a65487592c65f069bcb94d1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:03,622 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:03,622 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=12, startTime=1732124883468; duration=0sec 2024-11-20T17:48:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:03,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124943608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124943620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124943622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/264cbefe453a419ab7dcc701e1b2a3a6 is 50, key is test_row_0/B:col10/1732124882862/Put/seqid=0 2024-11-20T17:48:03,644 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:03,644 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/f583a2a25fdc4580afded3880971e4df is 50, key is test_row_0/C:col10/1732124882512/Put/seqid=0 2024-11-20T17:48:03,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741861_1037 (size=12001) 2024-11-20T17:48:03,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741862_1038 (size=12241) 2024-11-20T17:48:03,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,701 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/f583a2a25fdc4580afded3880971e4df as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f583a2a25fdc4580afded3880971e4df 2024-11-20T17:48:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124943700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,712 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into f583a2a25fdc4580afded3880971e4df(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:03,712 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:03,712 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=12, startTime=1732124883468; duration=0sec 2024-11-20T17:48:03,713 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:03,713 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:03,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124943727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124943727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124943738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124943738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:03,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:03,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:03,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:03,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124943937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124943947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124943948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T17:48:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124943957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/264cbefe453a419ab7dcc701e1b2a3a6 2024-11-20T17:48:04,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:04,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:04,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/7ccf765e04984e1fb9235cea974dc336 is 50, key is test_row_0/C:col10/1732124882862/Put/seqid=0 2024-11-20T17:48:04,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741863_1039 (size=12001) 2024-11-20T17:48:04,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/7ccf765e04984e1fb9235cea974dc336 2024-11-20T17:48:04,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/869dd43873c7436e87dfd4ffeefc62ab as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/869dd43873c7436e87dfd4ffeefc62ab 2024-11-20T17:48:04,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/869dd43873c7436e87dfd4ffeefc62ab, entries=200, sequenceid=127, filesize=14.0 K 2024-11-20T17:48:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/264cbefe453a419ab7dcc701e1b2a3a6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/264cbefe453a419ab7dcc701e1b2a3a6 2024-11-20T17:48:04,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/264cbefe453a419ab7dcc701e1b2a3a6, entries=150, sequenceid=127, filesize=11.7 K 2024-11-20T17:48:04,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/7ccf765e04984e1fb9235cea974dc336 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/7ccf765e04984e1fb9235cea974dc336 2024-11-20T17:48:04,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/7ccf765e04984e1fb9235cea974dc336, entries=150, sequenceid=127, filesize=11.7 K 2024-11-20T17:48:04,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f6561bfd37132223c1b60b9e3a6231d8 in 687ms, sequenceid=127, compaction requested=false 2024-11-20T17:48:04,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:04,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T17:48:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:04,239 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T17:48:04,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:04,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/55ab041cff914f8e8413729d8c7a6a82 is 50, key is test_row_0/A:col10/1732124883600/Put/seqid=0 2024-11-20T17:48:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124944275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124944280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741864_1040 (size=12151) 2024-11-20T17:48:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124944286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,296 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/55ab041cff914f8e8413729d8c7a6a82 2024-11-20T17:48:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124944293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/8d56e335d40c48bca23ac6d6c066c102 is 50, key is test_row_0/B:col10/1732124883600/Put/seqid=0 2024-11-20T17:48:04,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741865_1041 (size=12151) 2024-11-20T17:48:04,347 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/8d56e335d40c48bca23ac6d6c066c102 2024-11-20T17:48:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/3c0c91fa5c3e439ca02d971e433e1e40 is 50, key is test_row_0/C:col10/1732124883600/Put/seqid=0 2024-11-20T17:48:04,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741866_1042 (size=12151) 2024-11-20T17:48:04,393 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/3c0c91fa5c3e439ca02d971e433e1e40 2024-11-20T17:48:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124944397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124944397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124944407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124944408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/55ab041cff914f8e8413729d8c7a6a82 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55ab041cff914f8e8413729d8c7a6a82 2024-11-20T17:48:04,440 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55ab041cff914f8e8413729d8c7a6a82, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T17:48:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/8d56e335d40c48bca23ac6d6c066c102 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/8d56e335d40c48bca23ac6d6c066c102 2024-11-20T17:48:04,455 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/8d56e335d40c48bca23ac6d6c066c102, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T17:48:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/3c0c91fa5c3e439ca02d971e433e1e40 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/3c0c91fa5c3e439ca02d971e433e1e40 2024-11-20T17:48:04,473 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/3c0c91fa5c3e439ca02d971e433e1e40, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T17:48:04,475 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f6561bfd37132223c1b60b9e3a6231d8 in 235ms, sequenceid=156, compaction requested=true 2024-11-20T17:48:04,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:04,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:04,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T17:48:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T17:48:04,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T17:48:04,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6590 sec 2024-11-20T17:48:04,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6760 sec 2024-11-20T17:48:04,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:04,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:04,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/4d117233cbe24e81be17af2d02dc767b is 50, key is test_row_0/A:col10/1732124884278/Put/seqid=0 2024-11-20T17:48:04,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741867_1043 (size=14541) 2024-11-20T17:48:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124944659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124944662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124944668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124944669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124944708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,713 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in f6561bfd37132223c1b60b9e3a6231d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124944778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124944778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124944778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124944778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T17:48:04,978 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:04,979 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T17:48:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T17:48:04,983 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:04,984 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:04,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124944988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124944988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124944988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124944989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/4d117233cbe24e81be17af2d02dc767b 2024-11-20T17:48:05,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/cfee4d987f304e4691266cae1eecb473 is 50, key is test_row_0/B:col10/1732124884278/Put/seqid=0 2024-11-20T17:48:05,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741868_1044 (size=12151) 2024-11-20T17:48:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T17:48:05,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T17:48:05,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124945297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124945297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124945297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124945298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/cfee4d987f304e4691266cae1eecb473 2024-11-20T17:48:05,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/6ba2bfb1775743a6a644c728bbca05a2 is 50, key is test_row_0/C:col10/1732124884278/Put/seqid=0 2024-11-20T17:48:05,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741869_1045 (size=12151) 2024-11-20T17:48:05,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T17:48:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:05,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124945808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124945808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124945808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124945817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:05,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:05,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:05,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/6ba2bfb1775743a6a644c728bbca05a2 2024-11-20T17:48:05,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/4d117233cbe24e81be17af2d02dc767b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/4d117233cbe24e81be17af2d02dc767b 2024-11-20T17:48:05,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/4d117233cbe24e81be17af2d02dc767b, entries=200, sequenceid=167, filesize=14.2 K 2024-11-20T17:48:05,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/cfee4d987f304e4691266cae1eecb473 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/cfee4d987f304e4691266cae1eecb473 2024-11-20T17:48:05,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/cfee4d987f304e4691266cae1eecb473, entries=150, sequenceid=167, filesize=11.9 K 2024-11-20T17:48:05,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/6ba2bfb1775743a6a644c728bbca05a2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/6ba2bfb1775743a6a644c728bbca05a2 2024-11-20T17:48:06,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/6ba2bfb1775743a6a644c728bbca05a2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-20T17:48:06,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 1403ms, sequenceid=167, compaction requested=true 2024-11-20T17:48:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:06,015 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:06,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:06,015 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:06,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:06,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:06,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:06,020 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:06,020 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:06,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:06,020 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:06,020 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:06,020 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:06,020 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d894c04c40a040a1b630867cbdd15c6b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/869dd43873c7436e87dfd4ffeefc62ab, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55ab041cff914f8e8413729d8c7a6a82, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/4d117233cbe24e81be17af2d02dc767b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=52.0 K 2024-11-20T17:48:06,021 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:06,021 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/15a8cb6c4a65487592c65f069bcb94d1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/264cbefe453a419ab7dcc701e1b2a3a6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/8d56e335d40c48bca23ac6d6c066c102, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/cfee4d987f304e4691266cae1eecb473] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=47.4 K 2024-11-20T17:48:06,021 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d894c04c40a040a1b630867cbdd15c6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732124882507 2024-11-20T17:48:06,024 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a8cb6c4a65487592c65f069bcb94d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732124882507 2024-11-20T17:48:06,025 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 869dd43873c7436e87dfd4ffeefc62ab, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732124882860 2024-11-20T17:48:06,025 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 264cbefe453a419ab7dcc701e1b2a3a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732124882860 2024-11-20T17:48:06,026 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ab041cff914f8e8413729d8c7a6a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732124883600 2024-11-20T17:48:06,026 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d56e335d40c48bca23ac6d6c066c102, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732124883600 2024-11-20T17:48:06,026 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d117233cbe24e81be17af2d02dc767b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732124884275 2024-11-20T17:48:06,026 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting cfee4d987f304e4691266cae1eecb473, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732124884275 2024-11-20T17:48:06,056 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:06,057 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/30247de53a38442d822e5a84002da3ba is 50, key is test_row_0/B:col10/1732124884278/Put/seqid=0 2024-11-20T17:48:06,061 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:06,062 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d4c8db215f3a467ab922ee836e143cee is 50, key is test_row_0/A:col10/1732124884278/Put/seqid=0 2024-11-20T17:48:06,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T17:48:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:06,079 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:48:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/55477918d86b4ccaac76724f1a3598ee is 50, key is test_row_0/A:col10/1732124884656/Put/seqid=0 2024-11-20T17:48:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T17:48:06,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741870_1046 (size=12527) 2024-11-20T17:48:06,141 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/30247de53a38442d822e5a84002da3ba as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/30247de53a38442d822e5a84002da3ba 2024-11-20T17:48:06,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741871_1047 (size=12527) 2024-11-20T17:48:06,153 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into 30247de53a38442d822e5a84002da3ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:06,153 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:06,153 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=12, startTime=1732124886015; duration=0sec 2024-11-20T17:48:06,153 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:06,153 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:06,153 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:06,155 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:06,156 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:06,156 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:06,157 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f583a2a25fdc4580afded3880971e4df, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/7ccf765e04984e1fb9235cea974dc336, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/3c0c91fa5c3e439ca02d971e433e1e40, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/6ba2bfb1775743a6a644c728bbca05a2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=47.4 K 2024-11-20T17:48:06,159 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d4c8db215f3a467ab922ee836e143cee as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d4c8db215f3a467ab922ee836e143cee 2024-11-20T17:48:06,159 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f583a2a25fdc4580afded3880971e4df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732124882507 2024-11-20T17:48:06,160 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ccf765e04984e1fb9235cea974dc336, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732124882860 2024-11-20T17:48:06,160 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c0c91fa5c3e439ca02d971e433e1e40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732124883600 2024-11-20T17:48:06,161 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba2bfb1775743a6a644c728bbca05a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732124884275 2024-11-20T17:48:06,170 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into d4c8db215f3a467ab922ee836e143cee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:06,170 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:06,170 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=12, startTime=1732124886014; duration=0sec 2024-11-20T17:48:06,171 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:06,171 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:06,206 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:06,206 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/80f6a507024e40beb8543cc0d3a39832 is 50, key is test_row_0/C:col10/1732124884278/Put/seqid=0 2024-11-20T17:48:06,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741872_1048 (size=12151) 2024-11-20T17:48:06,213 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/55477918d86b4ccaac76724f1a3598ee 2024-11-20T17:48:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c78e65d8cbaf4177851c3c8a58a7f4f9 is 50, key is test_row_0/B:col10/1732124884656/Put/seqid=0 2024-11-20T17:48:06,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741873_1049 (size=12527) 2024-11-20T17:48:06,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741874_1050 (size=12151) 2024-11-20T17:48:06,264 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c78e65d8cbaf4177851c3c8a58a7f4f9 2024-11-20T17:48:06,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a226298ade484b1896e1666e1e9e22f9 is 50, key is test_row_0/C:col10/1732124884656/Put/seqid=0 2024-11-20T17:48:06,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741875_1051 (size=12151) 2024-11-20T17:48:06,316 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a226298ade484b1896e1666e1e9e22f9 2024-11-20T17:48:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/55477918d86b4ccaac76724f1a3598ee as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55477918d86b4ccaac76724f1a3598ee 2024-11-20T17:48:06,335 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55477918d86b4ccaac76724f1a3598ee, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T17:48:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c78e65d8cbaf4177851c3c8a58a7f4f9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c78e65d8cbaf4177851c3c8a58a7f4f9 2024-11-20T17:48:06,351 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c78e65d8cbaf4177851c3c8a58a7f4f9, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T17:48:06,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a226298ade484b1896e1666e1e9e22f9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a226298ade484b1896e1666e1e9e22f9 2024-11-20T17:48:06,373 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a226298ade484b1896e1666e1e9e22f9, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T17:48:06,376 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for f6561bfd37132223c1b60b9e3a6231d8 in 297ms, sequenceid=192, compaction requested=false 2024-11-20T17:48:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:06,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:06,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T17:48:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T17:48:06,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T17:48:06,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-20T17:48:06,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4110 sec 2024-11-20T17:48:06,676 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/80f6a507024e40beb8543cc0d3a39832 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/80f6a507024e40beb8543cc0d3a39832 2024-11-20T17:48:06,690 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 80f6a507024e40beb8543cc0d3a39832(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:06,691 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:06,691 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=12, startTime=1732124886019; duration=0sec 2024-11-20T17:48:06,691 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:06,693 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:06,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:06,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/be740e7381204d05a922b8d46b8b9fd1 is 50, key is test_row_0/A:col10/1732124886768/Put/seqid=0 2024-11-20T17:48:06,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741876_1052 (size=16931) 2024-11-20T17:48:06,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/be740e7381204d05a922b8d46b8b9fd1 2024-11-20T17:48:06,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5c410c26308346fdbc0640ac07bb0e13 is 50, key is test_row_0/B:col10/1732124886768/Put/seqid=0 2024-11-20T17:48:06,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741877_1053 (size=12151) 2024-11-20T17:48:06,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5c410c26308346fdbc0640ac07bb0e13 2024-11-20T17:48:06,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2f463812e2b04924954f2880b610641a is 50, key is test_row_0/C:col10/1732124886768/Put/seqid=0 2024-11-20T17:48:06,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124946865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:06,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124946871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:06,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741878_1054 (size=12151) 2024-11-20T17:48:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124946937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124946937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:06,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124946946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:06,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:06,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124947049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124947050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124947057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124947058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124947058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T17:48:07,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:07,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T17:48:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T17:48:07,136 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:07,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:07,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T17:48:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124947259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124947258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124947268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124947268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124947268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T17:48:07,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:07,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:07,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:07,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:07,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:07,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2f463812e2b04924954f2880b610641a 2024-11-20T17:48:07,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/be740e7381204d05a922b8d46b8b9fd1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/be740e7381204d05a922b8d46b8b9fd1 2024-11-20T17:48:07,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/be740e7381204d05a922b8d46b8b9fd1, entries=250, sequenceid=206, filesize=16.5 K 2024-11-20T17:48:07,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5c410c26308346fdbc0640ac07bb0e13 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5c410c26308346fdbc0640ac07bb0e13 2024-11-20T17:48:07,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5c410c26308346fdbc0640ac07bb0e13, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T17:48:07,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2f463812e2b04924954f2880b610641a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2f463812e2b04924954f2880b610641a 2024-11-20T17:48:07,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2f463812e2b04924954f2880b610641a, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T17:48:07,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 628ms, sequenceid=206, compaction requested=true 2024-11-20T17:48:07,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:07,397 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:07,399 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:07,399 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:07,399 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:07,399 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d4c8db215f3a467ab922ee836e143cee, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55477918d86b4ccaac76724f1a3598ee, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/be740e7381204d05a922b8d46b8b9fd1] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=40.6 K 2024-11-20T17:48:07,399 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:07,401 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:07,402 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:07,403 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:07,403 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/30247de53a38442d822e5a84002da3ba, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c78e65d8cbaf4177851c3c8a58a7f4f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5c410c26308346fdbc0640ac07bb0e13] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.0 K 2024-11-20T17:48:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:07,400 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c8db215f3a467ab922ee836e143cee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732124884275 2024-11-20T17:48:07,404 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 30247de53a38442d822e5a84002da3ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732124884275 2024-11-20T17:48:07,404 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55477918d86b4ccaac76724f1a3598ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732124884656 2024-11-20T17:48:07,404 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c78e65d8cbaf4177851c3c8a58a7f4f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732124884656 2024-11-20T17:48:07,405 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting be740e7381204d05a922b8d46b8b9fd1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124886745 2024-11-20T17:48:07,405 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c410c26308346fdbc0640ac07bb0e13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124886745 2024-11-20T17:48:07,426 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:07,426 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/f3ddd98a591d46c098fac7d8b8da7702 is 50, key is test_row_0/B:col10/1732124886768/Put/seqid=0 2024-11-20T17:48:07,438 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:07,439 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3f89ff0e1ad9438abfde993796260d35 is 50, key is test_row_0/A:col10/1732124886768/Put/seqid=0 2024-11-20T17:48:07,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T17:48:07,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:07,453 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:48:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T17:48:07,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741879_1055 (size=12629) 2024-11-20T17:48:07,510 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/f3ddd98a591d46c098fac7d8b8da7702 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/f3ddd98a591d46c098fac7d8b8da7702 2024-11-20T17:48:07,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741880_1056 (size=12629) 2024-11-20T17:48:07,520 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into f3ddd98a591d46c098fac7d8b8da7702(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:07,520 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:07,520 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124887399; duration=0sec 2024-11-20T17:48:07,521 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:07,521 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:07,521 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:07,523 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:07,523 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:07,523 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:07,523 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/80f6a507024e40beb8543cc0d3a39832, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a226298ade484b1896e1666e1e9e22f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2f463812e2b04924954f2880b610641a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.0 K 2024-11-20T17:48:07,524 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 80f6a507024e40beb8543cc0d3a39832, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732124884275 2024-11-20T17:48:07,525 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a226298ade484b1896e1666e1e9e22f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732124884656 2024-11-20T17:48:07,525 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f463812e2b04924954f2880b610641a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124886745 2024-11-20T17:48:07,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b1bf1b4846874217a204e5521dd16432 is 50, key is test_row_0/A:col10/1732124886879/Put/seqid=0 2024-11-20T17:48:07,555 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:07,555 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/70a610add0bf488496ded94f5b843f2f is 50, key is test_row_0/C:col10/1732124886768/Put/seqid=0 2024-11-20T17:48:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:07,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:07,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741881_1057 (size=12151) 2024-11-20T17:48:07,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124947583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124947584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124947585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124947587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124947587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741882_1058 (size=12629) 2024-11-20T17:48:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124947697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124947698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124947698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124947698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T17:48:07,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124947907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124947907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124947908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124947908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:07,929 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3f89ff0e1ad9438abfde993796260d35 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f89ff0e1ad9438abfde993796260d35 2024-11-20T17:48:07,949 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 3f89ff0e1ad9438abfde993796260d35(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:07,949 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:07,949 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124887397; duration=0sec 2024-11-20T17:48:07,950 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:07,950 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:07,976 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b1bf1b4846874217a204e5521dd16432 2024-11-20T17:48:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/844b10df8b4141259ad729cc3cb8b20d is 50, key is test_row_0/B:col10/1732124886879/Put/seqid=0 2024-11-20T17:48:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741883_1059 (size=12151) 2024-11-20T17:48:08,031 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/844b10df8b4141259ad729cc3cb8b20d 2024-11-20T17:48:08,031 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/70a610add0bf488496ded94f5b843f2f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/70a610add0bf488496ded94f5b843f2f 2024-11-20T17:48:08,041 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 70a610add0bf488496ded94f5b843f2f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:08,041 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:08,041 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=13, startTime=1732124887403; duration=0sec 2024-11-20T17:48:08,042 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:08,042 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/ef6aa64c253a4bf19df67eec1bfd8e97 is 50, key is test_row_0/C:col10/1732124886879/Put/seqid=0 2024-11-20T17:48:08,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741884_1060 (size=12151) 2024-11-20T17:48:08,072 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/ef6aa64c253a4bf19df67eec1bfd8e97 2024-11-20T17:48:08,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b1bf1b4846874217a204e5521dd16432 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b1bf1b4846874217a204e5521dd16432 2024-11-20T17:48:08,095 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b1bf1b4846874217a204e5521dd16432, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T17:48:08,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/844b10df8b4141259ad729cc3cb8b20d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/844b10df8b4141259ad729cc3cb8b20d 2024-11-20T17:48:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124948098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,109 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/844b10df8b4141259ad729cc3cb8b20d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T17:48:08,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/ef6aa64c253a4bf19df67eec1bfd8e97 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ef6aa64c253a4bf19df67eec1bfd8e97 2024-11-20T17:48:08,122 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ef6aa64c253a4bf19df67eec1bfd8e97, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T17:48:08,131 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f6561bfd37132223c1b60b9e3a6231d8 in 678ms, sequenceid=231, compaction requested=false 2024-11-20T17:48:08,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:08,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T17:48:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T17:48:08,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T17:48:08,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-20T17:48:08,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0100 sec 2024-11-20T17:48:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:08,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:08,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:48:08,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/636bbda2666642fc9654032fd02520f3 is 50, key is test_row_0/A:col10/1732124887581/Put/seqid=0 2024-11-20T17:48:08,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741885_1061 (size=12151) 2024-11-20T17:48:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T17:48:08,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:08,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T17:48:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T17:48:08,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:08,281 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:08,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:08,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124948311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124948311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124948313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124948313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T17:48:08,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124948427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124948427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124948427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124948427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T17:48:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T17:48:08,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:08,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T17:48:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124948638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124948638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124948640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124948640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/636bbda2666642fc9654032fd02520f3 2024-11-20T17:48:08,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/533838156c514d3786553834c396484b is 50, key is test_row_0/B:col10/1732124887581/Put/seqid=0 2024-11-20T17:48:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741886_1062 (size=12151) 2024-11-20T17:48:08,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/533838156c514d3786553834c396484b 2024-11-20T17:48:08,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/b1689f97a3d542be96308616098e0c6c is 50, key is test_row_0/C:col10/1732124887581/Put/seqid=0 2024-11-20T17:48:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741887_1063 (size=12151) 2024-11-20T17:48:08,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T17:48:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T17:48:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:08,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T17:48:08,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124948947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124948947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124948957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124948957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T17:48:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:09,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:09,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:09,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:09,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124949107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,109 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in f6561bfd37132223c1b60b9e3a6231d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/b1689f97a3d542be96308616098e0c6c 2024-11-20T17:48:09,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/636bbda2666642fc9654032fd02520f3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/636bbda2666642fc9654032fd02520f3 2024-11-20T17:48:09,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/636bbda2666642fc9654032fd02520f3, entries=150, sequenceid=247, filesize=11.9 K 2024-11-20T17:48:09,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/533838156c514d3786553834c396484b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/533838156c514d3786553834c396484b 2024-11-20T17:48:09,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/533838156c514d3786553834c396484b, entries=150, sequenceid=247, filesize=11.9 K 2024-11-20T17:48:09,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/b1689f97a3d542be96308616098e0c6c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/b1689f97a3d542be96308616098e0c6c 2024-11-20T17:48:09,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/b1689f97a3d542be96308616098e0c6c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-20T17:48:09,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 935ms, sequenceid=247, compaction requested=true 2024-11-20T17:48:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:09,156 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:09,156 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:09,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:09,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:09,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:09,158 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:09,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:09,158 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:09,158 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:09,158 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:09,158 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f89ff0e1ad9438abfde993796260d35, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b1bf1b4846874217a204e5521dd16432, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/636bbda2666642fc9654032fd02520f3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.1 K 2024-11-20T17:48:09,158 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/f3ddd98a591d46c098fac7d8b8da7702, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/844b10df8b4141259ad729cc3cb8b20d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/533838156c514d3786553834c396484b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.1 K 2024-11-20T17:48:09,158 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f89ff0e1ad9438abfde993796260d35, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124886745 2024-11-20T17:48:09,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ddd98a591d46c098fac7d8b8da7702, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124886745 2024-11-20T17:48:09,159 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 844b10df8b4141259ad729cc3cb8b20d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732124886864 2024-11-20T17:48:09,159 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1bf1b4846874217a204e5521dd16432, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732124886864 2024-11-20T17:48:09,159 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 533838156c514d3786553834c396484b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732124887581 2024-11-20T17:48:09,159 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636bbda2666642fc9654032fd02520f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732124887581 2024-11-20T17:48:09,170 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:09,171 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/065634fa7eed49038ad51925db2b3330 is 50, key is test_row_0/A:col10/1732124887581/Put/seqid=0 2024-11-20T17:48:09,171 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:09,172 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/bd4b023143d647c5b89a78fb6818d707 is 50, key is test_row_0/B:col10/1732124887581/Put/seqid=0 2024-11-20T17:48:09,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741888_1064 (size=12731) 2024-11-20T17:48:09,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741889_1065 (size=12731) 2024-11-20T17:48:09,204 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/065634fa7eed49038ad51925db2b3330 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/065634fa7eed49038ad51925db2b3330 2024-11-20T17:48:09,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T17:48:09,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:09,213 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:48:09,215 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 065634fa7eed49038ad51925db2b3330(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:09,216 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:09,217 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124889156; duration=0sec 2024-11-20T17:48:09,217 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:09,217 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:09,217 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:09,219 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:09,219 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:09,219 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:09,219 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/70a610add0bf488496ded94f5b843f2f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ef6aa64c253a4bf19df67eec1bfd8e97, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/b1689f97a3d542be96308616098e0c6c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.1 K 2024-11-20T17:48:09,220 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70a610add0bf488496ded94f5b843f2f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124886745 2024-11-20T17:48:09,220 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6aa64c253a4bf19df67eec1bfd8e97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732124886864 2024-11-20T17:48:09,221 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1689f97a3d542be96308616098e0c6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732124887581 2024-11-20T17:48:09,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b870369623e24b18bf0df5ea1331c952 is 50, key is test_row_0/A:col10/1732124888307/Put/seqid=0 2024-11-20T17:48:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741890_1066 (size=12301) 2024-11-20T17:48:09,235 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:09,235 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/88d94323fb144cbca7156d2463f08fac is 50, key is test_row_0/C:col10/1732124887581/Put/seqid=0 2024-11-20T17:48:09,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741891_1067 (size=12731) 2024-11-20T17:48:09,269 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/88d94323fb144cbca7156d2463f08fac as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/88d94323fb144cbca7156d2463f08fac 2024-11-20T17:48:09,276 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 88d94323fb144cbca7156d2463f08fac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:09,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:09,277 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=13, startTime=1732124889156; duration=0sec 2024-11-20T17:48:09,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:09,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T17:48:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:09,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124949469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124949470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124949488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124949488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124949597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124949597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124949597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124949597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,608 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/bd4b023143d647c5b89a78fb6818d707 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bd4b023143d647c5b89a78fb6818d707 2024-11-20T17:48:09,620 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into bd4b023143d647c5b89a78fb6818d707(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:09,620 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:09,620 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124889156; duration=0sec 2024-11-20T17:48:09,620 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:09,620 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:09,631 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b870369623e24b18bf0df5ea1331c952 2024-11-20T17:48:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/bcb2b0a8248d4580867e18a872aed864 is 50, key is test_row_0/B:col10/1732124888307/Put/seqid=0 2024-11-20T17:48:09,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741892_1068 (size=12301) 2024-11-20T17:48:09,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124949808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124949808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124949808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124949808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,048 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/bcb2b0a8248d4580867e18a872aed864 2024-11-20T17:48:10,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/8520a79b9f9940209871c037e38085d6 is 50, key is test_row_0/C:col10/1732124888307/Put/seqid=0 2024-11-20T17:48:10,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741893_1069 (size=12301) 2024-11-20T17:48:10,094 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/8520a79b9f9940209871c037e38085d6 2024-11-20T17:48:10,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b870369623e24b18bf0df5ea1331c952 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b870369623e24b18bf0df5ea1331c952 2024-11-20T17:48:10,115 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b870369623e24b18bf0df5ea1331c952, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T17:48:10,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/bcb2b0a8248d4580867e18a872aed864 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bcb2b0a8248d4580867e18a872aed864 2024-11-20T17:48:10,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124950118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124950119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124950118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,131 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bcb2b0a8248d4580867e18a872aed864, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T17:48:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/8520a79b9f9940209871c037e38085d6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8520a79b9f9940209871c037e38085d6 2024-11-20T17:48:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124950128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,141 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8520a79b9f9940209871c037e38085d6, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T17:48:10,143 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for f6561bfd37132223c1b60b9e3a6231d8 in 930ms, sequenceid=272, compaction requested=false 2024-11-20T17:48:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:10,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T17:48:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T17:48:10,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T17:48:10,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-11-20T17:48:10,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8720 sec 2024-11-20T17:48:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T17:48:10,429 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:10,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T17:48:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T17:48:10,433 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:10,434 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:10,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T17:48:10,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T17:48:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:10,587 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:48:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/5359dd26fe33476baec951ea2c05fb52 is 50, key is test_row_0/A:col10/1732124889469/Put/seqid=0 2024-11-20T17:48:10,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741894_1070 (size=12301) 2024-11-20T17:48:10,608 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/5359dd26fe33476baec951ea2c05fb52 2024-11-20T17:48:10,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c74263ecf9054cf0b9e26c063492158f is 50, key is test_row_0/B:col10/1732124889469/Put/seqid=0 2024-11-20T17:48:10,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:10,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741895_1071 (size=12301) 2024-11-20T17:48:10,661 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c74263ecf9054cf0b9e26c063492158f 2024-11-20T17:48:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/34268eaf0e544895b79c5e5cbf69f883 is 50, key is test_row_0/C:col10/1732124889469/Put/seqid=0 2024-11-20T17:48:10,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741896_1072 (size=12301) 2024-11-20T17:48:10,694 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/34268eaf0e544895b79c5e5cbf69f883 2024-11-20T17:48:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/5359dd26fe33476baec951ea2c05fb52 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/5359dd26fe33476baec951ea2c05fb52 2024-11-20T17:48:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124950701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124950701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124950703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124950705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,712 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/5359dd26fe33476baec951ea2c05fb52, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T17:48:10,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/c74263ecf9054cf0b9e26c063492158f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c74263ecf9054cf0b9e26c063492158f 2024-11-20T17:48:10,725 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c74263ecf9054cf0b9e26c063492158f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T17:48:10,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/34268eaf0e544895b79c5e5cbf69f883 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/34268eaf0e544895b79c5e5cbf69f883 2024-11-20T17:48:10,732 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/34268eaf0e544895b79c5e5cbf69f883, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T17:48:10,733 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 146ms, sequenceid=287, compaction requested=true 2024-11-20T17:48:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T17:48:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T17:48:10,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T17:48:10,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 301 msec 2024-11-20T17:48:10,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 310 msec 2024-11-20T17:48:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T17:48:10,748 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:10,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T17:48:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T17:48:10,752 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:10,755 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:10,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:10,821 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:10,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T17:48:10,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b8e5bb019719419f9789ecda7822efd4 is 50, key is test_row_0/A:col10/1732124890819/Put/seqid=0 2024-11-20T17:48:10,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124950837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124950836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741897_1073 (size=12301) 2024-11-20T17:48:10,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124950839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124950839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T17:48:10,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:10,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124950947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124950947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124950957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124950957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T17:48:11,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124951127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,129 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in f6561bfd37132223c1b60b9e3a6231d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124951157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124951157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124951168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124951168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b8e5bb019719419f9789ecda7822efd4 2024-11-20T17:48:11,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/63dfee28dbcc4be398fb3e051c9d995e is 50, key is test_row_0/B:col10/1732124890819/Put/seqid=0 2024-11-20T17:48:11,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741898_1074 (size=12301) 2024-11-20T17:48:11,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/63dfee28dbcc4be398fb3e051c9d995e 2024-11-20T17:48:11,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/f474688c5ae34125b18e8d7597d0e0d8 is 50, key is test_row_0/C:col10/1732124890819/Put/seqid=0 2024-11-20T17:48:11,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741899_1075 (size=12301) 2024-11-20T17:48:11,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:11,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T17:48:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124951468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124951468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124951477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124951478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:11,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:11,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/f474688c5ae34125b18e8d7597d0e0d8 2024-11-20T17:48:11,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/b8e5bb019719419f9789ecda7822efd4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b8e5bb019719419f9789ecda7822efd4 2024-11-20T17:48:11,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b8e5bb019719419f9789ecda7822efd4, entries=150, sequenceid=315, filesize=12.0 K 2024-11-20T17:48:11,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/63dfee28dbcc4be398fb3e051c9d995e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/63dfee28dbcc4be398fb3e051c9d995e 2024-11-20T17:48:11,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/63dfee28dbcc4be398fb3e051c9d995e, entries=150, sequenceid=315, filesize=12.0 K 2024-11-20T17:48:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/f474688c5ae34125b18e8d7597d0e0d8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f474688c5ae34125b18e8d7597d0e0d8 2024-11-20T17:48:11,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f474688c5ae34125b18e8d7597d0e0d8, entries=150, sequenceid=315, filesize=12.0 K 2024-11-20T17:48:11,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for f6561bfd37132223c1b60b9e3a6231d8 in 939ms, sequenceid=315, compaction requested=true 2024-11-20T17:48:11,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:11,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:11,760 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:11,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:11,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:11,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:11,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:11,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:48:11,761 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:11,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:11,763 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:11,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:11,763 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:11,763 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,763 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,764 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bd4b023143d647c5b89a78fb6818d707, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bcb2b0a8248d4580867e18a872aed864, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c74263ecf9054cf0b9e26c063492158f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/63dfee28dbcc4be398fb3e051c9d995e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=48.5 K 2024-11-20T17:48:11,764 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/065634fa7eed49038ad51925db2b3330, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b870369623e24b18bf0df5ea1331c952, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/5359dd26fe33476baec951ea2c05fb52, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b8e5bb019719419f9789ecda7822efd4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=48.5 K 2024-11-20T17:48:11,764 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting bd4b023143d647c5b89a78fb6818d707, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732124887581 2024-11-20T17:48:11,764 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065634fa7eed49038ad51925db2b3330, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732124887581 2024-11-20T17:48:11,765 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b870369623e24b18bf0df5ea1331c952, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732124888307 2024-11-20T17:48:11,765 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting bcb2b0a8248d4580867e18a872aed864, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732124888307 2024-11-20T17:48:11,767 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c74263ecf9054cf0b9e26c063492158f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732124889462 2024-11-20T17:48:11,767 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5359dd26fe33476baec951ea2c05fb52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732124889462 2024-11-20T17:48:11,768 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e5bb019719419f9789ecda7822efd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732124890818 2024-11-20T17:48:11,768 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 63dfee28dbcc4be398fb3e051c9d995e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732124890818 2024-11-20T17:48:11,800 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#63 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:11,801 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/00252bea17d84d5289c310a621a19979 is 50, key is test_row_0/A:col10/1732124890819/Put/seqid=0 2024-11-20T17:48:11,801 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:11,802 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/2cf990cc88784d3f9c3fcdd08da008b3 is 50, key is test_row_0/B:col10/1732124890819/Put/seqid=0 2024-11-20T17:48:11,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741900_1076 (size=13017) 2024-11-20T17:48:11,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741901_1077 (size=13017) 2024-11-20T17:48:11,827 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/2cf990cc88784d3f9c3fcdd08da008b3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/2cf990cc88784d3f9c3fcdd08da008b3 2024-11-20T17:48:11,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T17:48:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,836 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T17:48:11,838 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into 2cf990cc88784d3f9c3fcdd08da008b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:11,839 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:11,839 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=12, startTime=1732124891760; duration=0sec 2024-11-20T17:48:11,839 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:11,839 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:11,840 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3607531c1bd64753a80b8f5ed9bbe570 is 50, key is test_row_0/A:col10/1732124890835/Put/seqid=0 2024-11-20T17:48:11,843 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:11,843 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:11,843 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:11,844 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/88d94323fb144cbca7156d2463f08fac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8520a79b9f9940209871c037e38085d6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/34268eaf0e544895b79c5e5cbf69f883, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f474688c5ae34125b18e8d7597d0e0d8] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=48.5 K 2024-11-20T17:48:11,845 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d94323fb144cbca7156d2463f08fac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732124887581 2024-11-20T17:48:11,846 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8520a79b9f9940209871c037e38085d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732124888307 2024-11-20T17:48:11,846 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 34268eaf0e544895b79c5e5cbf69f883, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732124889462 2024-11-20T17:48:11,847 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f474688c5ae34125b18e8d7597d0e0d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732124890818 2024-11-20T17:48:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741902_1078 (size=12301) 2024-11-20T17:48:11,853 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3607531c1bd64753a80b8f5ed9bbe570 2024-11-20T17:48:11,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/be8c50cea63a4d48a91dd652718cd6b7 is 50, key is test_row_0/B:col10/1732124890835/Put/seqid=0 2024-11-20T17:48:11,882 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:11,883 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/c1d0ed5f4e1e44a583efca50549aeaac is 50, key is test_row_0/C:col10/1732124890819/Put/seqid=0 2024-11-20T17:48:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T17:48:11,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741903_1079 (size=12301) 2024-11-20T17:48:11,943 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/be8c50cea63a4d48a91dd652718cd6b7 2024-11-20T17:48:11,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741904_1080 (size=13017) 2024-11-20T17:48:11,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a0f85b0ae8c446279b9e37c177239537 is 50, key is test_row_0/C:col10/1732124890835/Put/seqid=0 2024-11-20T17:48:11,970 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/c1d0ed5f4e1e44a583efca50549aeaac as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c1d0ed5f4e1e44a583efca50549aeaac 2024-11-20T17:48:11,979 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into c1d0ed5f4e1e44a583efca50549aeaac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:11,979 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:11,979 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=12, startTime=1732124891761; duration=0sec 2024-11-20T17:48:11,979 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:11,979 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:11,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:11,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741905_1081 (size=12301) 2024-11-20T17:48:11,991 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a0f85b0ae8c446279b9e37c177239537 2024-11-20T17:48:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3607531c1bd64753a80b8f5ed9bbe570 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3607531c1bd64753a80b8f5ed9bbe570 2024-11-20T17:48:12,007 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3607531c1bd64753a80b8f5ed9bbe570, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T17:48:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/be8c50cea63a4d48a91dd652718cd6b7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/be8c50cea63a4d48a91dd652718cd6b7 2024-11-20T17:48:12,014 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/be8c50cea63a4d48a91dd652718cd6b7, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T17:48:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/a0f85b0ae8c446279b9e37c177239537 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a0f85b0ae8c446279b9e37c177239537 2024-11-20T17:48:12,023 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a0f85b0ae8c446279b9e37c177239537, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T17:48:12,024 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=134.18 KB/137400 for f6561bfd37132223c1b60b9e3a6231d8 in 188ms, sequenceid=323, compaction requested=false 2024-11-20T17:48:12,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:12,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:12,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T17:48:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:12,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:12,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:48:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T17:48:12,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T17:48:12,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-11-20T17:48:12,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/49eab56d2ca943fe8132542b7feb56a0 is 50, key is test_row_0/A:col10/1732124892018/Put/seqid=0 2024-11-20T17:48:12,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2810 sec 2024-11-20T17:48:12,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124952038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124952039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124952041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124952043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741906_1082 (size=22065) 2024-11-20T17:48:12,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/49eab56d2ca943fe8132542b7feb56a0 2024-11-20T17:48:12,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/1afb3c3634b042b6906fa078453a14fc is 50, key is test_row_0/B:col10/1732124892018/Put/seqid=0 2024-11-20T17:48:12,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741907_1083 (size=12301) 2024-11-20T17:48:12,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124952147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124952147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124952158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124952158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,224 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/00252bea17d84d5289c310a621a19979 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/00252bea17d84d5289c310a621a19979 2024-11-20T17:48:12,231 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 00252bea17d84d5289c310a621a19979(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:12,231 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:12,231 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=12, startTime=1732124891760; duration=0sec 2024-11-20T17:48:12,231 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:12,231 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:12,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124952357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124952357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124952367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124952367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/1afb3c3634b042b6906fa078453a14fc 2024-11-20T17:48:12,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/ec9ecf9ebd664e5d9612234f920e632a is 50, key is test_row_0/C:col10/1732124892018/Put/seqid=0 2024-11-20T17:48:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741908_1084 (size=12301) 2024-11-20T17:48:12,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/ec9ecf9ebd664e5d9612234f920e632a 2024-11-20T17:48:12,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/49eab56d2ca943fe8132542b7feb56a0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/49eab56d2ca943fe8132542b7feb56a0 2024-11-20T17:48:12,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/49eab56d2ca943fe8132542b7feb56a0, entries=350, sequenceid=349, filesize=21.5 K 2024-11-20T17:48:12,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/1afb3c3634b042b6906fa078453a14fc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1afb3c3634b042b6906fa078453a14fc 2024-11-20T17:48:12,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1afb3c3634b042b6906fa078453a14fc, entries=150, sequenceid=349, filesize=12.0 K 2024-11-20T17:48:12,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/ec9ecf9ebd664e5d9612234f920e632a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ec9ecf9ebd664e5d9612234f920e632a 2024-11-20T17:48:12,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ec9ecf9ebd664e5d9612234f920e632a, entries=150, sequenceid=349, filesize=12.0 K 2024-11-20T17:48:12,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f6561bfd37132223c1b60b9e3a6231d8 in 514ms, sequenceid=349, compaction requested=true 2024-11-20T17:48:12,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:12,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:12,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:12,540 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:12,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:12,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:12,540 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:12,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:12,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:12,541 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:12,541 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:12,541 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:12,541 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:12,541 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:12,541 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:12,541 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/2cf990cc88784d3f9c3fcdd08da008b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/be8c50cea63a4d48a91dd652718cd6b7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1afb3c3634b042b6906fa078453a14fc] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.7 K 2024-11-20T17:48:12,541 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/00252bea17d84d5289c310a621a19979, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3607531c1bd64753a80b8f5ed9bbe570, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/49eab56d2ca943fe8132542b7feb56a0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=46.3 K 2024-11-20T17:48:12,542 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf990cc88784d3f9c3fcdd08da008b3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732124890818 2024-11-20T17:48:12,542 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00252bea17d84d5289c310a621a19979, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732124890818 2024-11-20T17:48:12,542 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting be8c50cea63a4d48a91dd652718cd6b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732124890823 2024-11-20T17:48:12,544 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afb3c3634b042b6906fa078453a14fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732124892018 2024-11-20T17:48:12,544 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3607531c1bd64753a80b8f5ed9bbe570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732124890823 2024-11-20T17:48:12,545 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49eab56d2ca943fe8132542b7feb56a0, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732124892013 2024-11-20T17:48:12,561 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:12,561 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:12,561 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/e33119f331f64d469ee8b5a81b49f03d is 50, key is test_row_0/B:col10/1732124892018/Put/seqid=0 2024-11-20T17:48:12,561 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/c3c01186c1b74c068c0bf20517dfa831 is 50, key is test_row_0/A:col10/1732124892018/Put/seqid=0 2024-11-20T17:48:12,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741909_1085 (size=13119) 2024-11-20T17:48:12,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741910_1086 (size=13119) 2024-11-20T17:48:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:12,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:12,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T17:48:12,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3f5cb7721af94c7dae4dc7074ae3e74c is 50, key is test_row_0/A:col10/1732124892039/Put/seqid=0 2024-11-20T17:48:12,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741911_1087 (size=17181) 2024-11-20T17:48:12,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3f5cb7721af94c7dae4dc7074ae3e74c 2024-11-20T17:48:12,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/550fd72ca20448149441f06cc2b23601 is 50, key is test_row_0/B:col10/1732124892039/Put/seqid=0 2024-11-20T17:48:12,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741912_1088 (size=12301) 2024-11-20T17:48:12,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/550fd72ca20448149441f06cc2b23601 2024-11-20T17:48:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124952708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124952708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124952713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124952715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1 is 50, key is test_row_0/C:col10/1732124892039/Put/seqid=0 2024-11-20T17:48:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741913_1089 (size=12301) 2024-11-20T17:48:12,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1 2024-11-20T17:48:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/3f5cb7721af94c7dae4dc7074ae3e74c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f5cb7721af94c7dae4dc7074ae3e74c 2024-11-20T17:48:12,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f5cb7721af94c7dae4dc7074ae3e74c, entries=250, sequenceid=364, filesize=16.8 K 2024-11-20T17:48:12,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/550fd72ca20448149441f06cc2b23601 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/550fd72ca20448149441f06cc2b23601 2024-11-20T17:48:12,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/550fd72ca20448149441f06cc2b23601, entries=150, sequenceid=364, filesize=12.0 K 2024-11-20T17:48:12,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1 2024-11-20T17:48:12,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1, entries=150, sequenceid=364, filesize=12.0 K 2024-11-20T17:48:12,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f6561bfd37132223c1b60b9e3a6231d8 in 94ms, sequenceid=364, compaction requested=true 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:12,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T17:48:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:12,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:12,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:48:12,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/64887fb846f54485846a8c81246b83fb is 50, key is test_row_0/A:col10/1732124892700/Put/seqid=0 2024-11-20T17:48:12,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124952845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124952845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741914_1090 (size=14741) 2024-11-20T17:48:12,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/64887fb846f54485846a8c81246b83fb 2024-11-20T17:48:12,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124952846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124952847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/1c658ba8ef4b425ea52b2bf611a88c0e is 50, key is test_row_0/B:col10/1732124892700/Put/seqid=0 2024-11-20T17:48:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741915_1091 (size=12301) 2024-11-20T17:48:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T17:48:12,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:12,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T17:48:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T17:48:12,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:12,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:12,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124952957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124952957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124952967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124952967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:12,978 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/e33119f331f64d469ee8b5a81b49f03d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e33119f331f64d469ee8b5a81b49f03d 2024-11-20T17:48:12,991 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into e33119f331f64d469ee8b5a81b49f03d(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:12,991 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:12,991 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124892540; duration=0sec 2024-11-20T17:48:12,992 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T17:48:12,992 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:12,992 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-20T17:48:12,992 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:48:12,993 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:48:12,993 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. because compaction request was cancelled 2024-11-20T17:48:12,993 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:12,993 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:12,997 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/c3c01186c1b74c068c0bf20517dfa831 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/c3c01186c1b74c068c0bf20517dfa831 2024-11-20T17:48:12,998 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:12,998 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:12,998 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:12,998 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c1d0ed5f4e1e44a583efca50549aeaac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a0f85b0ae8c446279b9e37c177239537, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ec9ecf9ebd664e5d9612234f920e632a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=48.8 K 2024-11-20T17:48:12,999 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d0ed5f4e1e44a583efca50549aeaac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732124890818 2024-11-20T17:48:13,000 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f85b0ae8c446279b9e37c177239537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732124890823 2024-11-20T17:48:13,000 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9ecf9ebd664e5d9612234f920e632a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732124892018 2024-11-20T17:48:13,001 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b8c3fbeecdb4f83ab1dfcb05c9d25a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732124892039 2024-11-20T17:48:13,005 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into c3c01186c1b74c068c0bf20517dfa831(size=12.8 K), total size for store is 29.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:13,005 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124892539; duration=0sec 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. because compaction request was cancelled 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:13,005 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T17:48:13,007 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:48:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T17:48:13,007 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:48:13,007 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. because compaction request was cancelled 2024-11-20T17:48:13,008 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:13,028 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:13,029 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/15ceceb72a3b4d8495b004cde281d3a2 is 50, key is test_row_0/C:col10/1732124892039/Put/seqid=0 2024-11-20T17:48:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741916_1092 (size=13153) 2024-11-20T17:48:13,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T17:48:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124953168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124953168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124953177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124953177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T17:48:13,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:13,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T17:48:13,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/1c658ba8ef4b425ea52b2bf611a88c0e 2024-11-20T17:48:13,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/11c360786ac943deb832b1cc46f18172 is 50, key is test_row_0/C:col10/1732124892700/Put/seqid=0 2024-11-20T17:48:13,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741917_1093 (size=12301) 2024-11-20T17:48:13,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T17:48:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:13,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,449 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/15ceceb72a3b4d8495b004cde281d3a2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/15ceceb72a3b4d8495b004cde281d3a2 2024-11-20T17:48:13,456 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 15ceceb72a3b4d8495b004cde281d3a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:13,456 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:13,456 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=12, startTime=1732124892763; duration=0sec 2024-11-20T17:48:13,456 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:13,456 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124953477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124953477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124953487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124953487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:13,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T17:48:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T17:48:13,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T17:48:13,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:13,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/11c360786ac943deb832b1cc46f18172 2024-11-20T17:48:13,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/64887fb846f54485846a8c81246b83fb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/64887fb846f54485846a8c81246b83fb 2024-11-20T17:48:13,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/64887fb846f54485846a8c81246b83fb, entries=200, sequenceid=389, filesize=14.4 K 2024-11-20T17:48:13,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/1c658ba8ef4b425ea52b2bf611a88c0e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1c658ba8ef4b425ea52b2bf611a88c0e 2024-11-20T17:48:13,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1c658ba8ef4b425ea52b2bf611a88c0e, entries=150, sequenceid=389, filesize=12.0 K 2024-11-20T17:48:13,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/11c360786ac943deb832b1cc46f18172 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/11c360786ac943deb832b1cc46f18172 2024-11-20T17:48:13,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/11c360786ac943deb832b1cc46f18172, entries=150, sequenceid=389, filesize=12.0 K 2024-11-20T17:48:13,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f6561bfd37132223c1b60b9e3a6231d8 in 924ms, sequenceid=389, compaction requested=true 2024-11-20T17:48:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:13,754 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:13,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:13,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:13,755 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:13,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:13,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:13,756 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:13,756 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:13,756 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,757 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e33119f331f64d469ee8b5a81b49f03d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/550fd72ca20448149441f06cc2b23601, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1c658ba8ef4b425ea52b2bf611a88c0e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.8 K 2024-11-20T17:48:13,757 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:13,757 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e33119f331f64d469ee8b5a81b49f03d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732124892018 2024-11-20T17:48:13,757 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:13,757 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,757 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/c3c01186c1b74c068c0bf20517dfa831, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f5cb7721af94c7dae4dc7074ae3e74c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/64887fb846f54485846a8c81246b83fb] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=44.0 K 2024-11-20T17:48:13,757 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 550fd72ca20448149441f06cc2b23601, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732124892039 2024-11-20T17:48:13,758 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c01186c1b74c068c0bf20517dfa831, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732124892018 2024-11-20T17:48:13,758 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c658ba8ef4b425ea52b2bf611a88c0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732124892700 2024-11-20T17:48:13,758 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5cb7721af94c7dae4dc7074ae3e74c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732124892037 2024-11-20T17:48:13,758 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64887fb846f54485846a8c81246b83fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732124892700 2024-11-20T17:48:13,771 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:13,772 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/95bc16548c6744a692d3f7697e6c12aa is 50, key is test_row_0/B:col10/1732124892700/Put/seqid=0 2024-11-20T17:48:13,782 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:13,783 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/862c850ea47f4de5bdf139242660abdf is 50, key is test_row_0/A:col10/1732124892700/Put/seqid=0 2024-11-20T17:48:13,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741918_1094 (size=13221) 2024-11-20T17:48:13,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/95bc16548c6744a692d3f7697e6c12aa as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/95bc16548c6744a692d3f7697e6c12aa 2024-11-20T17:48:13,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T17:48:13,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:13,820 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:13,827 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into 95bc16548c6744a692d3f7697e6c12aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:13,827 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:13,827 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124893754; duration=0sec 2024-11-20T17:48:13,827 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:13,827 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:13,828 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T17:48:13,830 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:48:13,830 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:48:13,830 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. because compaction request was cancelled 2024-11-20T17:48:13,830 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/1e185adaa72b437ab8fffd4405c6cd8c is 50, key is test_row_0/A:col10/1732124892845/Put/seqid=0 2024-11-20T17:48:13,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741919_1095 (size=13221) 2024-11-20T17:48:13,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741920_1096 (size=12301) 2024-11-20T17:48:13,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:14,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T17:48:14,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124954030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124954030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124954032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124954037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124954147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124954147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124954147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124954147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,245 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/862c850ea47f4de5bdf139242660abdf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/862c850ea47f4de5bdf139242660abdf 2024-11-20T17:48:14,246 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/1e185adaa72b437ab8fffd4405c6cd8c 2024-11-20T17:48:14,258 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 862c850ea47f4de5bdf139242660abdf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:14,258 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:14,258 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124893754; duration=0sec 2024-11-20T17:48:14,258 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:14,258 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:14,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/ba263ca409ee45e7bf2ecd3b07fb7ae8 is 50, key is test_row_0/B:col10/1732124892845/Put/seqid=0 2024-11-20T17:48:14,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741921_1097 (size=12301) 2024-11-20T17:48:14,283 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/ba263ca409ee45e7bf2ecd3b07fb7ae8 2024-11-20T17:48:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/cc650a85cca3468ebd158874752364e2 is 50, key is test_row_0/C:col10/1732124892845/Put/seqid=0 2024-11-20T17:48:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741922_1098 (size=12301) 2024-11-20T17:48:14,317 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/cc650a85cca3468ebd158874752364e2 2024-11-20T17:48:14,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/1e185adaa72b437ab8fffd4405c6cd8c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/1e185adaa72b437ab8fffd4405c6cd8c 2024-11-20T17:48:14,338 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/1e185adaa72b437ab8fffd4405c6cd8c, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T17:48:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/ba263ca409ee45e7bf2ecd3b07fb7ae8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ba263ca409ee45e7bf2ecd3b07fb7ae8 2024-11-20T17:48:14,349 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ba263ca409ee45e7bf2ecd3b07fb7ae8, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T17:48:14,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/cc650a85cca3468ebd158874752364e2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/cc650a85cca3468ebd158874752364e2 2024-11-20T17:48:14,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124954357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124954357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124954357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124954358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,362 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/cc650a85cca3468ebd158874752364e2, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T17:48:14,364 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f6561bfd37132223c1b60b9e3a6231d8 in 543ms, sequenceid=403, compaction requested=true 2024-11-20T17:48:14,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:14,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:14,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T17:48:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T17:48:14,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T17:48:14,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4670 sec 2024-11-20T17:48:14,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4750 sec 2024-11-20T17:48:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:14,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:14,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T17:48:14,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/db9c7174118c4d4f8c2fcfd26ba890b5 is 50, key is test_row_0/A:col10/1732124894033/Put/seqid=0 2024-11-20T17:48:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124954675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124954676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124954677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741923_1099 (size=12301) 2024-11-20T17:48:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124954678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124954787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124954787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124954787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124954787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124954997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124954997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124954998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124954998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T17:48:15,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:15,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-20T17:48:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:15,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:15,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:15,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:15,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/db9c7174118c4d4f8c2fcfd26ba890b5 2024-11-20T17:48:15,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/d729516a5ad142b28cfec24f1904002b is 50, key is test_row_0/B:col10/1732124894033/Put/seqid=0 2024-11-20T17:48:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741924_1100 (size=12301) 2024-11-20T17:48:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52996 deadline: 1732124955167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,168 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in f6561bfd37132223c1b60b9e3a6231d8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T17:48:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:15,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124955307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124955308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124955308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124955308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T17:48:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:15,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:15,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/d729516a5ad142b28cfec24f1904002b 2024-11-20T17:48:15,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/55d6002e39d4433eac7e9bcb0c433654 is 50, key is test_row_0/C:col10/1732124894033/Put/seqid=0 2024-11-20T17:48:15,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T17:48:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741925_1101 (size=12301) 2024-11-20T17:48:15,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T17:48:15,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:15,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:15,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T17:48:15,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:15,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:15,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124955817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124955818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124955818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124955819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:15,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:15,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/55d6002e39d4433eac7e9bcb0c433654 2024-11-20T17:48:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/db9c7174118c4d4f8c2fcfd26ba890b5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/db9c7174118c4d4f8c2fcfd26ba890b5 2024-11-20T17:48:15,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/db9c7174118c4d4f8c2fcfd26ba890b5, entries=150, sequenceid=431, filesize=12.0 K 2024-11-20T17:48:15,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/d729516a5ad142b28cfec24f1904002b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d729516a5ad142b28cfec24f1904002b 2024-11-20T17:48:15,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d729516a5ad142b28cfec24f1904002b, entries=150, sequenceid=431, filesize=12.0 K 2024-11-20T17:48:15,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/55d6002e39d4433eac7e9bcb0c433654 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/55d6002e39d4433eac7e9bcb0c433654 2024-11-20T17:48:15,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/55d6002e39d4433eac7e9bcb0c433654, entries=150, sequenceid=431, filesize=12.0 K 2024-11-20T17:48:15,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f6561bfd37132223c1b60b9e3a6231d8 in 1286ms, sequenceid=431, compaction requested=true 2024-11-20T17:48:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:15,955 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:15,955 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:15,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:15,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:15,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:15,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:15,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:15,957 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,957 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/95bc16548c6744a692d3f7697e6c12aa, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ba263ca409ee45e7bf2ecd3b07fb7ae8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d729516a5ad142b28cfec24f1904002b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.9 K 2024-11-20T17:48:15,957 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:15,958 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:15,958 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,958 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/862c850ea47f4de5bdf139242660abdf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/1e185adaa72b437ab8fffd4405c6cd8c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/db9c7174118c4d4f8c2fcfd26ba890b5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=36.9 K 2024-11-20T17:48:15,959 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862c850ea47f4de5bdf139242660abdf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732124892700 2024-11-20T17:48:15,959 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 95bc16548c6744a692d3f7697e6c12aa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732124892700 2024-11-20T17:48:15,959 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e185adaa72b437ab8fffd4405c6cd8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124892840 2024-11-20T17:48:15,959 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ba263ca409ee45e7bf2ecd3b07fb7ae8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124892840 2024-11-20T17:48:15,960 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d729516a5ad142b28cfec24f1904002b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732124894029 2024-11-20T17:48:15,960 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9c7174118c4d4f8c2fcfd26ba890b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732124894029 2024-11-20T17:48:15,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T17:48:15,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:15,971 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T17:48:15,977 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:15,978 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:15,978 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/0106b1ecc5204e27ac775c525d0230f2 is 50, key is test_row_0/A:col10/1732124894033/Put/seqid=0 2024-11-20T17:48:15,978 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/33234efce36c4b39a26b359382b294d7 is 50, key is test_row_0/B:col10/1732124894033/Put/seqid=0 2024-11-20T17:48:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d91bd2815a96463bb3c425f029e0f0a6 is 50, key is test_row_0/A:col10/1732124894675/Put/seqid=0 2024-11-20T17:48:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741927_1103 (size=13323) 2024-11-20T17:48:16,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741926_1102 (size=13323) 2024-11-20T17:48:16,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741928_1104 (size=12301) 2024-11-20T17:48:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:16,436 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/0106b1ecc5204e27ac775c525d0230f2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/0106b1ecc5204e27ac775c525d0230f2 2024-11-20T17:48:16,442 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 0106b1ecc5204e27ac775c525d0230f2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:16,442 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:16,442 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124895955; duration=0sec 2024-11-20T17:48:16,443 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:16,443 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:16,443 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:48:16,444 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d91bd2815a96463bb3c425f029e0f0a6 2024-11-20T17:48:16,446 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:48:16,446 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:16,446 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:16,446 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/15ceceb72a3b4d8495b004cde281d3a2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/11c360786ac943deb832b1cc46f18172, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/cc650a85cca3468ebd158874752364e2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/55d6002e39d4433eac7e9bcb0c433654] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=48.9 K 2024-11-20T17:48:16,448 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15ceceb72a3b4d8495b004cde281d3a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732124892039 2024-11-20T17:48:16,449 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c360786ac943deb832b1cc46f18172, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732124892700 2024-11-20T17:48:16,452 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc650a85cca3468ebd158874752364e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124892840 2024-11-20T17:48:16,461 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/33234efce36c4b39a26b359382b294d7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/33234efce36c4b39a26b359382b294d7 2024-11-20T17:48:16,461 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d6002e39d4433eac7e9bcb0c433654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732124894029 2024-11-20T17:48:16,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/fa5adc6a00a04497abc0b7f5ed237138 is 50, key is test_row_0/B:col10/1732124894675/Put/seqid=0 2024-11-20T17:48:16,470 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into 33234efce36c4b39a26b359382b294d7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:16,470 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:16,470 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124895955; duration=0sec 2024-11-20T17:48:16,470 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:16,471 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:16,491 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:16,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741929_1105 (size=12301) 2024-11-20T17:48:16,491 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/1a1e6affae7b4df5b3ed67ec1f5b5565 is 50, key is test_row_0/C:col10/1732124894033/Put/seqid=0 2024-11-20T17:48:16,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741930_1106 (size=13289) 2024-11-20T17:48:16,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:16,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124956866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124956868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124956873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124956874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,934 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/fa5adc6a00a04497abc0b7f5ed237138 2024-11-20T17:48:16,942 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/1a1e6affae7b4df5b3ed67ec1f5b5565 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/1a1e6affae7b4df5b3ed67ec1f5b5565 2024-11-20T17:48:16,950 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 1a1e6affae7b4df5b3ed67ec1f5b5565(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:16,950 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:16,950 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=12, startTime=1732124895956; duration=0sec 2024-11-20T17:48:16,950 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:16,950 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/8a89a26879ed462bba87034462ba60a9 is 50, key is test_row_0/C:col10/1732124894675/Put/seqid=0 2024-11-20T17:48:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741931_1107 (size=12301) 2024-11-20T17:48:16,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124956977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124956977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124956987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124956987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124957187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:17,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124957197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124957197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124957197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,367 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/8a89a26879ed462bba87034462ba60a9 2024-11-20T17:48:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/d91bd2815a96463bb3c425f029e0f0a6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d91bd2815a96463bb3c425f029e0f0a6 2024-11-20T17:48:17,379 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d91bd2815a96463bb3c425f029e0f0a6, entries=150, sequenceid=441, filesize=12.0 K 2024-11-20T17:48:17,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/fa5adc6a00a04497abc0b7f5ed237138 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/fa5adc6a00a04497abc0b7f5ed237138 2024-11-20T17:48:17,386 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/fa5adc6a00a04497abc0b7f5ed237138, entries=150, sequenceid=441, filesize=12.0 K 2024-11-20T17:48:17,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/8a89a26879ed462bba87034462ba60a9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8a89a26879ed462bba87034462ba60a9 2024-11-20T17:48:17,396 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8a89a26879ed462bba87034462ba60a9, entries=150, sequenceid=441, filesize=12.0 K 2024-11-20T17:48:17,399 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for f6561bfd37132223c1b60b9e3a6231d8 in 1428ms, sequenceid=441, compaction requested=false 2024-11-20T17:48:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-20T17:48:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-20T17:48:17,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T17:48:17,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3500 sec 2024-11-20T17:48:17,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3560 sec 2024-11-20T17:48:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:17,499 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:17,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T17:48:17,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/54edec4aeb334c178c2539d7c18012e5 is 50, key is test_row_0/A:col10/1732124897497/Put/seqid=0 2024-11-20T17:48:17,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741932_1108 (size=14741) 2024-11-20T17:48:17,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124957520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124957520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124957521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124957522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124957627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124957627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124957627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124957627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124957837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124957837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124957837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124957838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:17,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/54edec4aeb334c178c2539d7c18012e5 2024-11-20T17:48:17,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/d042bcc7e2bc4e09b1c34caee487b7e5 is 50, key is test_row_0/B:col10/1732124897497/Put/seqid=0 2024-11-20T17:48:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741933_1109 (size=12301) 2024-11-20T17:48:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124958147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124958147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124958147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124958147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/d042bcc7e2bc4e09b1c34caee487b7e5 2024-11-20T17:48:18,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/efc64e2476f34dabadbe2fa35f91459b is 50, key is test_row_0/C:col10/1732124897497/Put/seqid=0 2024-11-20T17:48:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741934_1110 (size=12301) 2024-11-20T17:48:18,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732124958657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52992 deadline: 1732124958657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53002 deadline: 1732124958657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732124958658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:48:18,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/efc64e2476f34dabadbe2fa35f91459b 2024-11-20T17:48:18,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/54edec4aeb334c178c2539d7c18012e5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/54edec4aeb334c178c2539d7c18012e5 2024-11-20T17:48:18,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/54edec4aeb334c178c2539d7c18012e5, entries=200, sequenceid=472, filesize=14.4 K 2024-11-20T17:48:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/d042bcc7e2bc4e09b1c34caee487b7e5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d042bcc7e2bc4e09b1c34caee487b7e5 2024-11-20T17:48:18,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d042bcc7e2bc4e09b1c34caee487b7e5, entries=150, sequenceid=472, filesize=12.0 K 2024-11-20T17:48:18,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/efc64e2476f34dabadbe2fa35f91459b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/efc64e2476f34dabadbe2fa35f91459b 2024-11-20T17:48:18,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/efc64e2476f34dabadbe2fa35f91459b, entries=150, sequenceid=472, filesize=12.0 K 2024-11-20T17:48:18,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for f6561bfd37132223c1b60b9e3a6231d8 in 1277ms, sequenceid=472, compaction requested=true 2024-11-20T17:48:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:18,776 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:18,777 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f6561bfd37132223c1b60b9e3a6231d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:18,779 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:18,779 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:18,779 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/B is initiating minor compaction (all files) 2024-11-20T17:48:18,779 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/A is initiating minor compaction (all files) 2024-11-20T17:48:18,779 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/B in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:18,779 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/A in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:18,779 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/0106b1ecc5204e27ac775c525d0230f2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d91bd2815a96463bb3c425f029e0f0a6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/54edec4aeb334c178c2539d7c18012e5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=39.4 K 2024-11-20T17:48:18,779 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/33234efce36c4b39a26b359382b294d7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/fa5adc6a00a04497abc0b7f5ed237138, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d042bcc7e2bc4e09b1c34caee487b7e5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=37.0 K 2024-11-20T17:48:18,779 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 33234efce36c4b39a26b359382b294d7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732124894029 2024-11-20T17:48:18,779 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0106b1ecc5204e27ac775c525d0230f2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732124894029 2024-11-20T17:48:18,780 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting fa5adc6a00a04497abc0b7f5ed237138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732124894674 2024-11-20T17:48:18,780 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91bd2815a96463bb3c425f029e0f0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732124894674 2024-11-20T17:48:18,780 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54edec4aeb334c178c2539d7c18012e5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732124896862 2024-11-20T17:48:18,780 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d042bcc7e2bc4e09b1c34caee487b7e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732124896862 2024-11-20T17:48:18,790 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#B#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:18,791 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/133f1d0ab0b9421e8bc2209d92b254b1 is 50, key is test_row_0/B:col10/1732124897497/Put/seqid=0 2024-11-20T17:48:18,792 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:18,792 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/511b534b6b3e4590bc5e20cf337157fd is 50, key is test_row_0/A:col10/1732124897497/Put/seqid=0 2024-11-20T17:48:18,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741935_1111 (size=13425) 2024-11-20T17:48:18,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741936_1112 (size=13425) 2024-11-20T17:48:18,807 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/511b534b6b3e4590bc5e20cf337157fd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/511b534b6b3e4590bc5e20cf337157fd 2024-11-20T17:48:18,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/A of f6561bfd37132223c1b60b9e3a6231d8 into 511b534b6b3e4590bc5e20cf337157fd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:18,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/A, priority=13, startTime=1732124898776; duration=0sec 2024-11-20T17:48:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:A 2024-11-20T17:48:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:18,815 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:18,815 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): f6561bfd37132223c1b60b9e3a6231d8/C is initiating minor compaction (all files) 2024-11-20T17:48:18,816 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6561bfd37132223c1b60b9e3a6231d8/C in TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:18,816 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/1a1e6affae7b4df5b3ed67ec1f5b5565, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8a89a26879ed462bba87034462ba60a9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/efc64e2476f34dabadbe2fa35f91459b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp, totalSize=37.0 K 2024-11-20T17:48:18,818 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1e6affae7b4df5b3ed67ec1f5b5565, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732124894029 2024-11-20T17:48:18,818 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a89a26879ed462bba87034462ba60a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732124894674 2024-11-20T17:48:18,819 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc64e2476f34dabadbe2fa35f91459b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732124896862 2024-11-20T17:48:18,828 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6561bfd37132223c1b60b9e3a6231d8#C#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:18,829 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2664bcae98684bcca48e01d877114a65 is 50, key is test_row_0/C:col10/1732124897497/Put/seqid=0 2024-11-20T17:48:18,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741937_1113 (size=13391) 2024-11-20T17:48:19,202 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/133f1d0ab0b9421e8bc2209d92b254b1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/133f1d0ab0b9421e8bc2209d92b254b1 2024-11-20T17:48:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T17:48:19,207 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:19,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:19,210 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/B of f6561bfd37132223c1b60b9e3a6231d8 into 133f1d0ab0b9421e8bc2209d92b254b1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:19,211 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:19,211 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/B, priority=13, startTime=1732124898776; duration=0sec 2024-11-20T17:48:19,211 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:19,211 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:B 2024-11-20T17:48:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-20T17:48:19,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T17:48:19,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:19,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:19,244 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/2664bcae98684bcca48e01d877114a65 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2664bcae98684bcca48e01d877114a65 2024-11-20T17:48:19,253 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f6561bfd37132223c1b60b9e3a6231d8/C of f6561bfd37132223c1b60b9e3a6231d8 into 2664bcae98684bcca48e01d877114a65(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:19,253 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:19,253 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., storeName=f6561bfd37132223c1b60b9e3a6231d8/C, priority=13, startTime=1732124898777; duration=0sec 2024-11-20T17:48:19,253 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:19,253 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6561bfd37132223c1b60b9e3a6231d8:C 2024-11-20T17:48:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T17:48:19,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T17:48:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:19,367 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T17:48:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/a1b85ee3bdbc41dda5c48d01839a6ab4 is 50, key is test_row_0/A:col10/1732124897519/Put/seqid=0 2024-11-20T17:48:19,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741938_1114 (size=9857) 2024-11-20T17:48:19,490 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T17:48:19,490 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,491 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T17:48:19,491 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,491 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T17:48:19,491 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,494 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T17:48:19,494 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,495 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,495 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,495 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,495 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,496 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,496 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,496 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,496 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T17:48:19,670 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T17:48:19,670 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,670 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. as already flushing 2024-11-20T17:48:19,670 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,670 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T17:48:19,670 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,670 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,670 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,670 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T17:48:19,671 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,671 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,671 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,679 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T17:48:19,679 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:19,679 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,679 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:19,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:19,778 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/a1b85ee3bdbc41dda5c48d01839a6ab4 2024-11-20T17:48:19,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/072ba3e5c4484f8bbde535592a9dc205 is 50, key is test_row_0/B:col10/1732124897519/Put/seqid=0 2024-11-20T17:48:19,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741939_1115 (size=9857) 2024-11-20T17:48:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T17:48:20,191 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/072ba3e5c4484f8bbde535592a9dc205 2024-11-20T17:48:20,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/9f8eaf4250734975b431cb48524420f0 is 50, key is test_row_0/C:col10/1732124897519/Put/seqid=0 2024-11-20T17:48:20,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741940_1116 (size=9857) 2024-11-20T17:48:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T17:48:20,602 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/9f8eaf4250734975b431cb48524420f0 2024-11-20T17:48:20,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/a1b85ee3bdbc41dda5c48d01839a6ab4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a1b85ee3bdbc41dda5c48d01839a6ab4 2024-11-20T17:48:20,613 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a1b85ee3bdbc41dda5c48d01839a6ab4, entries=100, sequenceid=483, filesize=9.6 K 2024-11-20T17:48:20,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/072ba3e5c4484f8bbde535592a9dc205 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/072ba3e5c4484f8bbde535592a9dc205 2024-11-20T17:48:20,620 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/072ba3e5c4484f8bbde535592a9dc205, entries=100, sequenceid=483, filesize=9.6 K 2024-11-20T17:48:20,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/9f8eaf4250734975b431cb48524420f0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/9f8eaf4250734975b431cb48524420f0 2024-11-20T17:48:20,625 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/9f8eaf4250734975b431cb48524420f0, entries=100, sequenceid=483, filesize=9.6 K 2024-11-20T17:48:20,626 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=26.84 KB/27480 for f6561bfd37132223c1b60b9e3a6231d8 in 1259ms, sequenceid=483, compaction requested=false 2024-11-20T17:48:20,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for f6561bfd37132223c1b60b9e3a6231d8: 2024-11-20T17:48:20,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:20,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-20T17:48:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-20T17:48:20,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T17:48:20,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-20T17:48:20,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.4230 sec 2024-11-20T17:48:21,177 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:48:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T17:48:21,359 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:25,210 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T17:48:25,210 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:25,210 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:25,211 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:25,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:25,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T17:48:25,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-20T17:48:25,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-20T17:48:25,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-20T17:48:25,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-20T17:48:25,211 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3901 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4099 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1747 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5237 rows 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1752 2024-11-20T17:48:25,212 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5253 rows 2024-11-20T17:48:25,213 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:48:25,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:48:25,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:25,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:25,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:25,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T17:48:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T17:48:25,226 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124905226"}]},"ts":"1732124905226"} 2024-11-20T17:48:25,228 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T17:48:25,229 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T17:48:25,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:48:25,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, UNASSIGN}] 2024-11-20T17:48:25,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, UNASSIGN 2024-11-20T17:48:25,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f6561bfd37132223c1b60b9e3a6231d8, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:25,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, UNASSIGN because future has completed 2024-11-20T17:48:25,242 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:48:25,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:48:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T17:48:25,399 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:25,399 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:48:25,399 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing f6561bfd37132223c1b60b9e3a6231d8, disabling compactions & flushes 2024-11-20T17:48:25,400 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:25,400 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:25,400 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. after waiting 0 ms 2024-11-20T17:48:25,400 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:25,400 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing f6561bfd37132223c1b60b9e3a6231d8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T17:48:25,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/648ee0c375ba491c90af1afe89b0adc3 is 50, key is test_row_1/A:col10/1732124899678/Put/seqid=0 2024-11-20T17:48:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741941_1117 (size=9857) 2024-11-20T17:48:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T17:48:25,809 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/648ee0c375ba491c90af1afe89b0adc3 2024-11-20T17:48:25,817 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5a2cace9e0f54b8892ff347f7de296e9 is 50, key is test_row_1/B:col10/1732124899678/Put/seqid=0 2024-11-20T17:48:25,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741942_1118 (size=9857) 2024-11-20T17:48:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T17:48:26,221 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5a2cace9e0f54b8892ff347f7de296e9 2024-11-20T17:48:26,229 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/96b532245de7462093b3a1cd5b61650c is 50, key is test_row_1/C:col10/1732124899678/Put/seqid=0 2024-11-20T17:48:26,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741943_1119 (size=9857) 2024-11-20T17:48:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T17:48:26,634 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/96b532245de7462093b3a1cd5b61650c 2024-11-20T17:48:26,646 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/A/648ee0c375ba491c90af1afe89b0adc3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/648ee0c375ba491c90af1afe89b0adc3 2024-11-20T17:48:26,654 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/648ee0c375ba491c90af1afe89b0adc3, entries=100, sequenceid=491, filesize=9.6 K 2024-11-20T17:48:26,655 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/B/5a2cace9e0f54b8892ff347f7de296e9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5a2cace9e0f54b8892ff347f7de296e9 2024-11-20T17:48:26,660 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5a2cace9e0f54b8892ff347f7de296e9, entries=100, sequenceid=491, filesize=9.6 K 2024-11-20T17:48:26,662 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/.tmp/C/96b532245de7462093b3a1cd5b61650c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/96b532245de7462093b3a1cd5b61650c 2024-11-20T17:48:26,667 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/96b532245de7462093b3a1cd5b61650c, entries=100, sequenceid=491, filesize=9.6 K 2024-11-20T17:48:26,668 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f6561bfd37132223c1b60b9e3a6231d8 in 1267ms, sequenceid=491, compaction requested=true 2024-11-20T17:48:26,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a71b5caa62254dab9ff7b71e505a5c28, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/872af24220bd4aabb6b633efe43032d0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/77e3408d65604dfdb8e7b2685f08eed2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/78c179f3a35e4798a834171b29e2e055, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/9ab8a2f2b39147f5b0345dd9c0fefd06, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d213a36d10a24fb7b5e2cfa976c05874, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b03651d7f8264e89a560438a8771de7e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d894c04c40a040a1b630867cbdd15c6b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/869dd43873c7436e87dfd4ffeefc62ab, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55ab041cff914f8e8413729d8c7a6a82, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/4d117233cbe24e81be17af2d02dc767b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d4c8db215f3a467ab922ee836e143cee, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55477918d86b4ccaac76724f1a3598ee, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/be740e7381204d05a922b8d46b8b9fd1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f89ff0e1ad9438abfde993796260d35, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b1bf1b4846874217a204e5521dd16432, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/065634fa7eed49038ad51925db2b3330, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/636bbda2666642fc9654032fd02520f3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b870369623e24b18bf0df5ea1331c952, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/5359dd26fe33476baec951ea2c05fb52, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/00252bea17d84d5289c310a621a19979, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b8e5bb019719419f9789ecda7822efd4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3607531c1bd64753a80b8f5ed9bbe570, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/49eab56d2ca943fe8132542b7feb56a0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/c3c01186c1b74c068c0bf20517dfa831, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f5cb7721af94c7dae4dc7074ae3e74c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/64887fb846f54485846a8c81246b83fb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/862c850ea47f4de5bdf139242660abdf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/1e185adaa72b437ab8fffd4405c6cd8c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/0106b1ecc5204e27ac775c525d0230f2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/db9c7174118c4d4f8c2fcfd26ba890b5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d91bd2815a96463bb3c425f029e0f0a6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/54edec4aeb334c178c2539d7c18012e5] to archive 2024-11-20T17:48:26,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:48:26,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a71b5caa62254dab9ff7b71e505a5c28 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a71b5caa62254dab9ff7b71e505a5c28 2024-11-20T17:48:26,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/872af24220bd4aabb6b633efe43032d0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/872af24220bd4aabb6b633efe43032d0 2024-11-20T17:48:26,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/77e3408d65604dfdb8e7b2685f08eed2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/77e3408d65604dfdb8e7b2685f08eed2 2024-11-20T17:48:26,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/78c179f3a35e4798a834171b29e2e055 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/78c179f3a35e4798a834171b29e2e055 2024-11-20T17:48:26,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/9ab8a2f2b39147f5b0345dd9c0fefd06 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/9ab8a2f2b39147f5b0345dd9c0fefd06 2024-11-20T17:48:26,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d213a36d10a24fb7b5e2cfa976c05874 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d213a36d10a24fb7b5e2cfa976c05874 2024-11-20T17:48:26,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b03651d7f8264e89a560438a8771de7e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b03651d7f8264e89a560438a8771de7e 2024-11-20T17:48:26,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d894c04c40a040a1b630867cbdd15c6b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d894c04c40a040a1b630867cbdd15c6b 2024-11-20T17:48:26,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/869dd43873c7436e87dfd4ffeefc62ab to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/869dd43873c7436e87dfd4ffeefc62ab 2024-11-20T17:48:26,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55ab041cff914f8e8413729d8c7a6a82 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55ab041cff914f8e8413729d8c7a6a82 2024-11-20T17:48:26,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/4d117233cbe24e81be17af2d02dc767b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/4d117233cbe24e81be17af2d02dc767b 2024-11-20T17:48:26,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d4c8db215f3a467ab922ee836e143cee to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d4c8db215f3a467ab922ee836e143cee 2024-11-20T17:48:26,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55477918d86b4ccaac76724f1a3598ee to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/55477918d86b4ccaac76724f1a3598ee 2024-11-20T17:48:26,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/be740e7381204d05a922b8d46b8b9fd1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/be740e7381204d05a922b8d46b8b9fd1 2024-11-20T17:48:26,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f89ff0e1ad9438abfde993796260d35 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f89ff0e1ad9438abfde993796260d35 2024-11-20T17:48:26,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b1bf1b4846874217a204e5521dd16432 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b1bf1b4846874217a204e5521dd16432 2024-11-20T17:48:26,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/065634fa7eed49038ad51925db2b3330 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/065634fa7eed49038ad51925db2b3330 2024-11-20T17:48:26,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/636bbda2666642fc9654032fd02520f3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/636bbda2666642fc9654032fd02520f3 2024-11-20T17:48:26,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b870369623e24b18bf0df5ea1331c952 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b870369623e24b18bf0df5ea1331c952 2024-11-20T17:48:26,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/5359dd26fe33476baec951ea2c05fb52 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/5359dd26fe33476baec951ea2c05fb52 2024-11-20T17:48:26,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/00252bea17d84d5289c310a621a19979 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/00252bea17d84d5289c310a621a19979 2024-11-20T17:48:26,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b8e5bb019719419f9789ecda7822efd4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/b8e5bb019719419f9789ecda7822efd4 2024-11-20T17:48:26,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3607531c1bd64753a80b8f5ed9bbe570 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3607531c1bd64753a80b8f5ed9bbe570 2024-11-20T17:48:26,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/49eab56d2ca943fe8132542b7feb56a0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/49eab56d2ca943fe8132542b7feb56a0 2024-11-20T17:48:26,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/c3c01186c1b74c068c0bf20517dfa831 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/c3c01186c1b74c068c0bf20517dfa831 2024-11-20T17:48:26,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f5cb7721af94c7dae4dc7074ae3e74c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/3f5cb7721af94c7dae4dc7074ae3e74c 2024-11-20T17:48:26,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/64887fb846f54485846a8c81246b83fb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/64887fb846f54485846a8c81246b83fb 2024-11-20T17:48:26,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/862c850ea47f4de5bdf139242660abdf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/862c850ea47f4de5bdf139242660abdf 2024-11-20T17:48:26,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/1e185adaa72b437ab8fffd4405c6cd8c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/1e185adaa72b437ab8fffd4405c6cd8c 2024-11-20T17:48:26,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/0106b1ecc5204e27ac775c525d0230f2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/0106b1ecc5204e27ac775c525d0230f2 2024-11-20T17:48:26,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/db9c7174118c4d4f8c2fcfd26ba890b5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/db9c7174118c4d4f8c2fcfd26ba890b5 2024-11-20T17:48:26,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d91bd2815a96463bb3c425f029e0f0a6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/d91bd2815a96463bb3c425f029e0f0a6 2024-11-20T17:48:26,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/54edec4aeb334c178c2539d7c18012e5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/54edec4aeb334c178c2539d7c18012e5 2024-11-20T17:48:26,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/931da03cc60141a5b1288f78b5b8054c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/dc967efb93a34519896e52c8c65e83a6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c50eb8ddd28c4af8892b2b2901ed3fcb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5ba87428b4ba42f69218ce5cf3e08c69, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e55647bbb3f94d688a9520cfe13a231d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/15a8cb6c4a65487592c65f069bcb94d1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/97ba47a7bb6049b39782ee27c8739c56, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/264cbefe453a419ab7dcc701e1b2a3a6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/8d56e335d40c48bca23ac6d6c066c102, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/30247de53a38442d822e5a84002da3ba, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/cfee4d987f304e4691266cae1eecb473, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c78e65d8cbaf4177851c3c8a58a7f4f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/f3ddd98a591d46c098fac7d8b8da7702, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5c410c26308346fdbc0640ac07bb0e13, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/844b10df8b4141259ad729cc3cb8b20d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bd4b023143d647c5b89a78fb6818d707, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/533838156c514d3786553834c396484b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bcb2b0a8248d4580867e18a872aed864, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c74263ecf9054cf0b9e26c063492158f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/2cf990cc88784d3f9c3fcdd08da008b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/63dfee28dbcc4be398fb3e051c9d995e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/be8c50cea63a4d48a91dd652718cd6b7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e33119f331f64d469ee8b5a81b49f03d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1afb3c3634b042b6906fa078453a14fc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/550fd72ca20448149441f06cc2b23601, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/95bc16548c6744a692d3f7697e6c12aa, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1c658ba8ef4b425ea52b2bf611a88c0e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ba263ca409ee45e7bf2ecd3b07fb7ae8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/33234efce36c4b39a26b359382b294d7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d729516a5ad142b28cfec24f1904002b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/fa5adc6a00a04497abc0b7f5ed237138, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d042bcc7e2bc4e09b1c34caee487b7e5] to archive 2024-11-20T17:48:26,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:48:26,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/931da03cc60141a5b1288f78b5b8054c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/931da03cc60141a5b1288f78b5b8054c 2024-11-20T17:48:26,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/dc967efb93a34519896e52c8c65e83a6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/dc967efb93a34519896e52c8c65e83a6 2024-11-20T17:48:26,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ff65fa1bdcc94dd6a94ac3605a7d5bfe 2024-11-20T17:48:26,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c50eb8ddd28c4af8892b2b2901ed3fcb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c50eb8ddd28c4af8892b2b2901ed3fcb 2024-11-20T17:48:26,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5ba87428b4ba42f69218ce5cf3e08c69 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5ba87428b4ba42f69218ce5cf3e08c69 2024-11-20T17:48:26,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e55647bbb3f94d688a9520cfe13a231d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e55647bbb3f94d688a9520cfe13a231d 2024-11-20T17:48:26,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/15a8cb6c4a65487592c65f069bcb94d1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/15a8cb6c4a65487592c65f069bcb94d1 2024-11-20T17:48:26,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/97ba47a7bb6049b39782ee27c8739c56 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/97ba47a7bb6049b39782ee27c8739c56 2024-11-20T17:48:26,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/264cbefe453a419ab7dcc701e1b2a3a6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/264cbefe453a419ab7dcc701e1b2a3a6 2024-11-20T17:48:26,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/8d56e335d40c48bca23ac6d6c066c102 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/8d56e335d40c48bca23ac6d6c066c102 2024-11-20T17:48:26,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/30247de53a38442d822e5a84002da3ba to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/30247de53a38442d822e5a84002da3ba 2024-11-20T17:48:26,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/cfee4d987f304e4691266cae1eecb473 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/cfee4d987f304e4691266cae1eecb473 2024-11-20T17:48:26,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c78e65d8cbaf4177851c3c8a58a7f4f9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c78e65d8cbaf4177851c3c8a58a7f4f9 2024-11-20T17:48:26,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/f3ddd98a591d46c098fac7d8b8da7702 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/f3ddd98a591d46c098fac7d8b8da7702 2024-11-20T17:48:26,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5c410c26308346fdbc0640ac07bb0e13 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5c410c26308346fdbc0640ac07bb0e13 2024-11-20T17:48:26,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/844b10df8b4141259ad729cc3cb8b20d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/844b10df8b4141259ad729cc3cb8b20d 2024-11-20T17:48:26,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bd4b023143d647c5b89a78fb6818d707 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bd4b023143d647c5b89a78fb6818d707 2024-11-20T17:48:26,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/533838156c514d3786553834c396484b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/533838156c514d3786553834c396484b 2024-11-20T17:48:26,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bcb2b0a8248d4580867e18a872aed864 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/bcb2b0a8248d4580867e18a872aed864 2024-11-20T17:48:26,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c74263ecf9054cf0b9e26c063492158f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/c74263ecf9054cf0b9e26c063492158f 2024-11-20T17:48:26,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/2cf990cc88784d3f9c3fcdd08da008b3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/2cf990cc88784d3f9c3fcdd08da008b3 2024-11-20T17:48:26,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/63dfee28dbcc4be398fb3e051c9d995e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/63dfee28dbcc4be398fb3e051c9d995e 2024-11-20T17:48:26,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/be8c50cea63a4d48a91dd652718cd6b7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/be8c50cea63a4d48a91dd652718cd6b7 2024-11-20T17:48:26,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e33119f331f64d469ee8b5a81b49f03d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/e33119f331f64d469ee8b5a81b49f03d 2024-11-20T17:48:26,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1afb3c3634b042b6906fa078453a14fc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1afb3c3634b042b6906fa078453a14fc 2024-11-20T17:48:26,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/550fd72ca20448149441f06cc2b23601 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/550fd72ca20448149441f06cc2b23601 2024-11-20T17:48:26,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/95bc16548c6744a692d3f7697e6c12aa to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/95bc16548c6744a692d3f7697e6c12aa 2024-11-20T17:48:26,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1c658ba8ef4b425ea52b2bf611a88c0e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/1c658ba8ef4b425ea52b2bf611a88c0e 2024-11-20T17:48:26,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ba263ca409ee45e7bf2ecd3b07fb7ae8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/ba263ca409ee45e7bf2ecd3b07fb7ae8 2024-11-20T17:48:26,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/33234efce36c4b39a26b359382b294d7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/33234efce36c4b39a26b359382b294d7 2024-11-20T17:48:26,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d729516a5ad142b28cfec24f1904002b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d729516a5ad142b28cfec24f1904002b 2024-11-20T17:48:26,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/fa5adc6a00a04497abc0b7f5ed237138 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/fa5adc6a00a04497abc0b7f5ed237138 2024-11-20T17:48:26,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d042bcc7e2bc4e09b1c34caee487b7e5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/d042bcc7e2bc4e09b1c34caee487b7e5 2024-11-20T17:48:26,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2d1d57042b1f4c78a1357bf90cca29e0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a94a16d5f2a9477e8e5767c6b428aa07, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/44727a01ff6941c4831678ca864da11e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c54d0658826d4bb99b40e004ae10d822, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/349e46a2380b421f95a721084762bdd9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2928cd17970f45e8938afd410f52a3dd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f583a2a25fdc4580afded3880971e4df, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/5e806049c05648d18b22fc5e415709b4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/7ccf765e04984e1fb9235cea974dc336, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/3c0c91fa5c3e439ca02d971e433e1e40, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/80f6a507024e40beb8543cc0d3a39832, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/6ba2bfb1775743a6a644c728bbca05a2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a226298ade484b1896e1666e1e9e22f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/70a610add0bf488496ded94f5b843f2f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2f463812e2b04924954f2880b610641a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ef6aa64c253a4bf19df67eec1bfd8e97, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/88d94323fb144cbca7156d2463f08fac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/b1689f97a3d542be96308616098e0c6c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8520a79b9f9940209871c037e38085d6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/34268eaf0e544895b79c5e5cbf69f883, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c1d0ed5f4e1e44a583efca50549aeaac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f474688c5ae34125b18e8d7597d0e0d8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a0f85b0ae8c446279b9e37c177239537, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ec9ecf9ebd664e5d9612234f920e632a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/15ceceb72a3b4d8495b004cde281d3a2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/11c360786ac943deb832b1cc46f18172, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/cc650a85cca3468ebd158874752364e2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/1a1e6affae7b4df5b3ed67ec1f5b5565, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/55d6002e39d4433eac7e9bcb0c433654, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8a89a26879ed462bba87034462ba60a9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/efc64e2476f34dabadbe2fa35f91459b] to archive 2024-11-20T17:48:26,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:48:26,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2d1d57042b1f4c78a1357bf90cca29e0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2d1d57042b1f4c78a1357bf90cca29e0 2024-11-20T17:48:26,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a94a16d5f2a9477e8e5767c6b428aa07 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a94a16d5f2a9477e8e5767c6b428aa07 2024-11-20T17:48:26,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/44727a01ff6941c4831678ca864da11e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/44727a01ff6941c4831678ca864da11e 2024-11-20T17:48:26,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c54d0658826d4bb99b40e004ae10d822 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c54d0658826d4bb99b40e004ae10d822 2024-11-20T17:48:26,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/349e46a2380b421f95a721084762bdd9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/349e46a2380b421f95a721084762bdd9 2024-11-20T17:48:26,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2928cd17970f45e8938afd410f52a3dd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2928cd17970f45e8938afd410f52a3dd 2024-11-20T17:48:26,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f583a2a25fdc4580afded3880971e4df to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f583a2a25fdc4580afded3880971e4df 2024-11-20T17:48:26,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/5e806049c05648d18b22fc5e415709b4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/5e806049c05648d18b22fc5e415709b4 2024-11-20T17:48:26,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/7ccf765e04984e1fb9235cea974dc336 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/7ccf765e04984e1fb9235cea974dc336 2024-11-20T17:48:26,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/3c0c91fa5c3e439ca02d971e433e1e40 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/3c0c91fa5c3e439ca02d971e433e1e40 2024-11-20T17:48:26,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/80f6a507024e40beb8543cc0d3a39832 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/80f6a507024e40beb8543cc0d3a39832 2024-11-20T17:48:26,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/6ba2bfb1775743a6a644c728bbca05a2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/6ba2bfb1775743a6a644c728bbca05a2 2024-11-20T17:48:26,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a226298ade484b1896e1666e1e9e22f9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a226298ade484b1896e1666e1e9e22f9 2024-11-20T17:48:26,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/70a610add0bf488496ded94f5b843f2f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/70a610add0bf488496ded94f5b843f2f 2024-11-20T17:48:26,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2f463812e2b04924954f2880b610641a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2f463812e2b04924954f2880b610641a 2024-11-20T17:48:26,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ef6aa64c253a4bf19df67eec1bfd8e97 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ef6aa64c253a4bf19df67eec1bfd8e97 2024-11-20T17:48:26,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/88d94323fb144cbca7156d2463f08fac to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/88d94323fb144cbca7156d2463f08fac 2024-11-20T17:48:26,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/b1689f97a3d542be96308616098e0c6c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/b1689f97a3d542be96308616098e0c6c 2024-11-20T17:48:26,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8520a79b9f9940209871c037e38085d6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8520a79b9f9940209871c037e38085d6 2024-11-20T17:48:26,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/34268eaf0e544895b79c5e5cbf69f883 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/34268eaf0e544895b79c5e5cbf69f883 2024-11-20T17:48:26,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c1d0ed5f4e1e44a583efca50549aeaac to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/c1d0ed5f4e1e44a583efca50549aeaac 2024-11-20T17:48:26,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f474688c5ae34125b18e8d7597d0e0d8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/f474688c5ae34125b18e8d7597d0e0d8 2024-11-20T17:48:26,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a0f85b0ae8c446279b9e37c177239537 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/a0f85b0ae8c446279b9e37c177239537 2024-11-20T17:48:26,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ec9ecf9ebd664e5d9612234f920e632a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/ec9ecf9ebd664e5d9612234f920e632a 2024-11-20T17:48:26,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/15ceceb72a3b4d8495b004cde281d3a2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/15ceceb72a3b4d8495b004cde281d3a2 2024-11-20T17:48:26,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/0b8c3fbeecdb4f83ab1dfcb05c9d25a1 2024-11-20T17:48:26,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/11c360786ac943deb832b1cc46f18172 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/11c360786ac943deb832b1cc46f18172 2024-11-20T17:48:26,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/cc650a85cca3468ebd158874752364e2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/cc650a85cca3468ebd158874752364e2 2024-11-20T17:48:26,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/1a1e6affae7b4df5b3ed67ec1f5b5565 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/1a1e6affae7b4df5b3ed67ec1f5b5565 2024-11-20T17:48:26,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/55d6002e39d4433eac7e9bcb0c433654 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/55d6002e39d4433eac7e9bcb0c433654 2024-11-20T17:48:26,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8a89a26879ed462bba87034462ba60a9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/8a89a26879ed462bba87034462ba60a9 2024-11-20T17:48:26,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/efc64e2476f34dabadbe2fa35f91459b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/efc64e2476f34dabadbe2fa35f91459b 2024-11-20T17:48:26,816 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/recovered.edits/494.seqid, newMaxSeqId=494, maxSeqId=1 2024-11-20T17:48:26,816 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. 2024-11-20T17:48:26,817 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for f6561bfd37132223c1b60b9e3a6231d8: Waiting for close lock at 1732124905399Running coprocessor pre-close hooks at 1732124905399Disabling compacts and flushes for region at 1732124905399Disabling writes for close at 1732124905400 (+1 ms)Obtaining lock to block concurrent updates at 1732124905400Preparing flush snapshotting stores in f6561bfd37132223c1b60b9e3a6231d8 at 1732124905400Finished memstore snapshotting TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732124905400Flushing stores of TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8. at 1732124905401 (+1 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/A: creating writer at 1732124905401Flushing f6561bfd37132223c1b60b9e3a6231d8/A: appending metadata at 1732124905404 (+3 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/A: closing flushed file at 1732124905404Flushing f6561bfd37132223c1b60b9e3a6231d8/B: creating writer at 1732124905813 (+409 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/B: appending metadata at 1732124905816 (+3 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/B: closing flushed file at 1732124905817 (+1 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/C: creating writer at 1732124906226 (+409 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/C: appending metadata at 1732124906228 (+2 ms)Flushing f6561bfd37132223c1b60b9e3a6231d8/C: closing flushed file at 1732124906228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bb6f14: reopening flushed file at 1732124906645 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ece3378: reopening flushed file at 1732124906654 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@80aa4c1: reopening flushed file at 1732124906661 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f6561bfd37132223c1b60b9e3a6231d8 in 1267ms, sequenceid=491, compaction requested=true at 1732124906668 (+7 ms)Writing region close event to WAL at 1732124906813 (+145 ms)Running coprocessor post-close hooks at 1732124906816 (+3 ms)Closed at 1732124906816 2024-11-20T17:48:26,819 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:26,819 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f6561bfd37132223c1b60b9e3a6231d8, regionState=CLOSED 2024-11-20T17:48:26,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:48:26,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T17:48:26,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure f6561bfd37132223c1b60b9e3a6231d8, server=613581fcdc40,44957,1732124872893 in 1.5800 sec 2024-11-20T17:48:26,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T17:48:26,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f6561bfd37132223c1b60b9e3a6231d8, UNASSIGN in 1.5880 sec 2024-11-20T17:48:26,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T17:48:26,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5980 sec 2024-11-20T17:48:26,832 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124906831"}]},"ts":"1732124906831"} 2024-11-20T17:48:26,833 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T17:48:26,834 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T17:48:26,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6140 sec 2024-11-20T17:48:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T17:48:27,377 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:27,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T17:48:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,384 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,386 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T17:48:27,390 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:27,394 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/recovered.edits] 2024-11-20T17:48:27,397 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/511b534b6b3e4590bc5e20cf337157fd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/511b534b6b3e4590bc5e20cf337157fd 2024-11-20T17:48:27,398 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/648ee0c375ba491c90af1afe89b0adc3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/648ee0c375ba491c90af1afe89b0adc3 2024-11-20T17:48:27,399 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a1b85ee3bdbc41dda5c48d01839a6ab4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/A/a1b85ee3bdbc41dda5c48d01839a6ab4 2024-11-20T17:48:27,401 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/072ba3e5c4484f8bbde535592a9dc205 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/072ba3e5c4484f8bbde535592a9dc205 2024-11-20T17:48:27,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/133f1d0ab0b9421e8bc2209d92b254b1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/133f1d0ab0b9421e8bc2209d92b254b1 2024-11-20T17:48:27,403 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5a2cace9e0f54b8892ff347f7de296e9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/B/5a2cace9e0f54b8892ff347f7de296e9 2024-11-20T17:48:27,406 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2664bcae98684bcca48e01d877114a65 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/2664bcae98684bcca48e01d877114a65 2024-11-20T17:48:27,407 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/96b532245de7462093b3a1cd5b61650c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/96b532245de7462093b3a1cd5b61650c 2024-11-20T17:48:27,408 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/9f8eaf4250734975b431cb48524420f0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/C/9f8eaf4250734975b431cb48524420f0 2024-11-20T17:48:27,410 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/recovered.edits/494.seqid to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8/recovered.edits/494.seqid 2024-11-20T17:48:27,411 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/f6561bfd37132223c1b60b9e3a6231d8 2024-11-20T17:48:27,411 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T17:48:27,416 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T17:48:27,424 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T17:48:27,455 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T17:48:27,457 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,457 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T17:48:27,458 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732124907457"}]},"ts":"9223372036854775807"} 2024-11-20T17:48:27,464 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T17:48:27,464 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f6561bfd37132223c1b60b9e3a6231d8, NAME => 'TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T17:48:27,464 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T17:48:27,465 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732124907464"}]},"ts":"9223372036854775807"} 2024-11-20T17:48:27,470 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T17:48:27,472 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-20T17:48:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T17:48:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T17:48:27,498 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:27,509 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-141067869_22 at /127.0.0.1:59770 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/613581fcdc40:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;613581fcdc40:44957-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 392) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8483 (was 9209) 2024-11-20T17:48:27,517 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=8483 2024-11-20T17:48:27,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:48:27,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:48:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:27,522 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:48:27,522 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:27,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-20T17:48:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T17:48:27,523 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:48:27,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741944_1120 (size=959) 2024-11-20T17:48:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T17:48:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T17:48:27,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:48:27,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741945_1121 (size=53) 2024-11-20T17:48:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T17:48:28,339 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:48:28,339 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee28898c42e6b66b22f1f209861bee42, disabling compactions & flushes 2024-11-20T17:48:28,339 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,339 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,339 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. after waiting 0 ms 2024-11-20T17:48:28,339 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,339 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,339 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee28898c42e6b66b22f1f209861bee42: Waiting for close lock at 1732124908339Disabling compacts and flushes for region at 1732124908339Disabling writes for close at 1732124908339Writing region close event to WAL at 1732124908339Closed at 1732124908339 2024-11-20T17:48:28,340 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:48:28,341 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732124908341"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732124908341"}]},"ts":"1732124908341"} 2024-11-20T17:48:28,343 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:48:28,344 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:48:28,344 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124908344"}]},"ts":"1732124908344"} 2024-11-20T17:48:28,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T17:48:28,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, ASSIGN}] 2024-11-20T17:48:28,347 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, ASSIGN 2024-11-20T17:48:28,348 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, ASSIGN; state=OFFLINE, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=false 2024-11-20T17:48:28,499 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:28,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, ASSIGN because future has completed 2024-11-20T17:48:28,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:48:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T17:48:28,659 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,659 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:48:28,660 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,660 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:48:28,660 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,661 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,663 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,667 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee28898c42e6b66b22f1f209861bee42 columnFamilyName A 2024-11-20T17:48:28,667 DEBUG [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:28,668 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(327): Store=ee28898c42e6b66b22f1f209861bee42/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:48:28,668 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,669 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee28898c42e6b66b22f1f209861bee42 columnFamilyName B 2024-11-20T17:48:28,669 DEBUG [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:28,670 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(327): Store=ee28898c42e6b66b22f1f209861bee42/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:48:28,670 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,671 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee28898c42e6b66b22f1f209861bee42 columnFamilyName C 2024-11-20T17:48:28,671 DEBUG [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:28,672 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(327): Store=ee28898c42e6b66b22f1f209861bee42/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:48:28,672 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,672 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,673 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,674 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,674 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,674 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:48:28,675 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,677 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:48:28,677 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ee28898c42e6b66b22f1f209861bee42; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70911007, jitterRate=0.05665634572505951}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:48:28,678 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:28,679 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ee28898c42e6b66b22f1f209861bee42: Running coprocessor pre-open hook at 1732124908661Writing region info on filesystem at 1732124908661Initializing all the Stores at 1732124908662 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124908662Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124908663 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124908663Cleaning up temporary data from old regions at 1732124908674 (+11 ms)Running coprocessor post-open hooks at 1732124908678 (+4 ms)Region opened successfully at 1732124908678 2024-11-20T17:48:28,679 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., pid=36, masterSystemTime=1732124908653 2024-11-20T17:48:28,681 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,682 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:28,682 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:28,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:48:28,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T17:48:28,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 in 184 msec 2024-11-20T17:48:28,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-20T17:48:28,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, ASSIGN in 341 msec 2024-11-20T17:48:28,690 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:48:28,690 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124908690"}]},"ts":"1732124908690"} 2024-11-20T17:48:28,692 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T17:48:28,693 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:48:28,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-20T17:48:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T17:48:29,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:29,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:29,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:29,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:29,671 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279c0d05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:29,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:29,676 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:29,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:29,683 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:48:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:48:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47569338 2024-11-20T17:48:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:48:29,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:48:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:48:29,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:48:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:29,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741946_1122 (size=995) 2024-11-20T17:48:30,109 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T17:48:30,109 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T17:48:30,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:48:30,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, REOPEN/MOVE}] 2024-11-20T17:48:30,119 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, REOPEN/MOVE 2024-11-20T17:48:30,120 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, REOPEN/MOVE because future has completed 2024-11-20T17:48:30,122 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:48:30,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:48:30,276 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,276 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:48:30,276 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing ee28898c42e6b66b22f1f209861bee42, disabling compactions & flushes 2024-11-20T17:48:30,276 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,276 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,276 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. after waiting 0 ms 2024-11-20T17:48:30,276 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,283 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:48:30,283 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,284 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for ee28898c42e6b66b22f1f209861bee42: Waiting for close lock at 1732124910276Running coprocessor pre-close hooks at 1732124910276Disabling compacts and flushes for region at 1732124910276Disabling writes for close at 1732124910276Writing region close event to WAL at 1732124910277 (+1 ms)Running coprocessor post-close hooks at 1732124910283 (+6 ms)Closed at 1732124910283 2024-11-20T17:48:30,284 WARN [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: ee28898c42e6b66b22f1f209861bee42 to self. 2024-11-20T17:48:30,286 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,286 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=CLOSED 2024-11-20T17:48:30,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:48:30,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-20T17:48:30,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 in 168 msec 2024-11-20T17:48:30,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, REOPEN/MOVE; state=CLOSED, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=true 2024-11-20T17:48:30,443 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, REOPEN/MOVE because future has completed 2024-11-20T17:48:30,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:48:30,603 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,603 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:48:30,604 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,604 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:48:30,604 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,604 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,609 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,616 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee28898c42e6b66b22f1f209861bee42 columnFamilyName A 2024-11-20T17:48:30,618 DEBUG [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:30,618 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(327): Store=ee28898c42e6b66b22f1f209861bee42/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:48:30,619 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,620 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee28898c42e6b66b22f1f209861bee42 columnFamilyName B 2024-11-20T17:48:30,620 DEBUG [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:30,620 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(327): Store=ee28898c42e6b66b22f1f209861bee42/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:48:30,620 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,621 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee28898c42e6b66b22f1f209861bee42 columnFamilyName C 2024-11-20T17:48:30,621 DEBUG [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:30,621 INFO [StoreOpener-ee28898c42e6b66b22f1f209861bee42-1 {}] regionserver.HStore(327): Store=ee28898c42e6b66b22f1f209861bee42/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:48:30,621 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,622 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,623 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,624 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,624 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,625 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:48:30,626 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,627 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened ee28898c42e6b66b22f1f209861bee42; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59664457, jitterRate=-0.11093030869960785}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:48:30,627 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,630 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for ee28898c42e6b66b22f1f209861bee42: Running coprocessor pre-open hook at 1732124910604Writing region info on filesystem at 1732124910604Initializing all the Stores at 1732124910605 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124910605Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124910606 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124910606Cleaning up temporary data from old regions at 1732124910624 (+18 ms)Running coprocessor post-open hooks at 1732124910627 (+3 ms)Region opened successfully at 1732124910630 (+3 ms) 2024-11-20T17:48:30,631 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., pid=41, masterSystemTime=1732124910597 2024-11-20T17:48:30,632 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,633 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,633 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=OPEN, openSeqNum=5, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:48:30,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-20T17:48:30,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 in 189 msec 2024-11-20T17:48:30,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T17:48:30,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, REOPEN/MOVE in 519 msec 2024-11-20T17:48:30,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-20T17:48:30,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-20T17:48:30,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-20T17:48:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-20T17:48:30,646 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:30,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,648 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0e0213, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f321aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22adaf42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,656 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300f21f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1525b6d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f67e621, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0ebdd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbdd797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec180b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,667 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173a0257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3bb9bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27005e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40682de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4444597e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,675 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c6f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:48:30,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:48:30,677 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:48:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:48:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:48:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217b1540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:48:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:48:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:30,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:48:30,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:48:30,680 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,680 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,681 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T17:48:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T17:48:30,683 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:30,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:30,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,696 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,698 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,699 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,709 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,710 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:30,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:30,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,718 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:48:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:48:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:48:30,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:48:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:48:30,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ddd179e78a94ff18ee59d23dabcd8ae_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124910705/Put/seqid=0 2024-11-20T17:48:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T17:48:30,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741947_1123 (size=14594) 2024-11-20T17:48:30,802 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:30,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124970791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ddd179e78a94ff18ee59d23dabcd8ae_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ddd179e78a94ff18ee59d23dabcd8ae_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:30,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/580ddb2af54141fd924b30a6fd5d3cfb, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:30,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124970801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124970801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124970804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124970804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/580ddb2af54141fd924b30a6fd5d3cfb is 175, key is test_row_0/A:col10/1732124910705/Put/seqid=0 2024-11-20T17:48:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741948_1124 (size=39549) 2024-11-20T17:48:30,834 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/580ddb2af54141fd924b30a6fd5d3cfb 2024-11-20T17:48:30,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:30,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:30,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/c2688ca1cc9b40f59af7678d02db277d is 50, key is test_row_0/B:col10/1732124910705/Put/seqid=0 2024-11-20T17:48:30,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741949_1125 (size=12001) 2024-11-20T17:48:30,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124970908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124970918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124970919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124970919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124970920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:30,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:30,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:30,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:30,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T17:48:31,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124971127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124971127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124971138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124971138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124971138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/c2688ca1cc9b40f59af7678d02db277d 2024-11-20T17:48:31,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:31,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T17:48:31,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/cf2ea60f46344714b4a47126ddfdd942 is 50, key is test_row_0/C:col10/1732124910705/Put/seqid=0 2024-11-20T17:48:31,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741950_1126 (size=12001) 2024-11-20T17:48:31,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124971447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124971447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124971448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124971448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124971457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/cf2ea60f46344714b4a47126ddfdd942 2024-11-20T17:48:31,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/580ddb2af54141fd924b30a6fd5d3cfb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb 2024-11-20T17:48:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb, entries=200, sequenceid=15, filesize=38.6 K 2024-11-20T17:48:31,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/c2688ca1cc9b40f59af7678d02db277d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c2688ca1cc9b40f59af7678d02db277d 2024-11-20T17:48:31,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:31,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c2688ca1cc9b40f59af7678d02db277d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T17:48:31,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/cf2ea60f46344714b4a47126ddfdd942 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cf2ea60f46344714b4a47126ddfdd942 2024-11-20T17:48:31,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cf2ea60f46344714b4a47126ddfdd942, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T17:48:31,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for ee28898c42e6b66b22f1f209861bee42 in 1076ms, sequenceid=15, compaction requested=false 2024-11-20T17:48:31,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T17:48:31,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T17:48:31,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:31,922 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T17:48:31,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120909a65bb4c234273b0ead24d46d27750_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124910801/Put/seqid=0 2024-11-20T17:48:31,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741951_1127 (size=12154) 2024-11-20T17:48:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:31,958 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120909a65bb4c234273b0ead24d46d27750_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120909a65bb4c234273b0ead24d46d27750_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:31,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:31,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/09753210871e424f9076d5e329a68276, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/09753210871e424f9076d5e329a68276 is 175, key is test_row_0/A:col10/1732124910801/Put/seqid=0 2024-11-20T17:48:31,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124971965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124971967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124971967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124971967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:31,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124971971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741952_1128 (size=30955) 2024-11-20T17:48:31,983 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/09753210871e424f9076d5e329a68276 2024-11-20T17:48:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/226298bff2c44383885ab5c532390f4b is 50, key is test_row_0/B:col10/1732124910801/Put/seqid=0 2024-11-20T17:48:32,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741953_1129 (size=12001) 2024-11-20T17:48:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124972077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124972078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124972078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124972288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124972288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124972288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,365 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:48:32,410 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/226298bff2c44383885ab5c532390f4b 2024-11-20T17:48:32,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/82feefe28c4e4ae29c487dd84ad01c28 is 50, key is test_row_0/C:col10/1732124910801/Put/seqid=0 2024-11-20T17:48:32,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741954_1130 (size=12001) 2024-11-20T17:48:32,460 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/82feefe28c4e4ae29c487dd84ad01c28 2024-11-20T17:48:32,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/09753210871e424f9076d5e329a68276 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276 2024-11-20T17:48:32,476 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276, entries=150, sequenceid=41, filesize=30.2 K 2024-11-20T17:48:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/226298bff2c44383885ab5c532390f4b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/226298bff2c44383885ab5c532390f4b 2024-11-20T17:48:32,497 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/226298bff2c44383885ab5c532390f4b, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T17:48:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/82feefe28c4e4ae29c487dd84ad01c28 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/82feefe28c4e4ae29c487dd84ad01c28 2024-11-20T17:48:32,512 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/82feefe28c4e4ae29c487dd84ad01c28, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T17:48:32,515 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ee28898c42e6b66b22f1f209861bee42 in 593ms, sequenceid=41, compaction requested=false 2024-11-20T17:48:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T17:48:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T17:48:32,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T17:48:32,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8330 sec 2024-11-20T17:48:32,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8390 sec 2024-11-20T17:48:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:32,609 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:32,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:32,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112057b1b72fad6f4e4b9e0ac2dfa4501897_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:32,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741955_1131 (size=14594) 2024-11-20T17:48:32,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:32,701 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112057b1b72fad6f4e4b9e0ac2dfa4501897_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112057b1b72fad6f4e4b9e0ac2dfa4501897_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:32,703 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/6845d4e978ba4e8686aceb1badee7d36, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:32,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/6845d4e978ba4e8686aceb1badee7d36 is 175, key is test_row_0/A:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:32,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124972699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124972702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124972710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741956_1132 (size=39549) 2024-11-20T17:48:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124972817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124972817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T17:48:32,828 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:32,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T17:48:32,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T17:48:32,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:32,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:32,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124972828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T17:48:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124972977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,981 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee28898c42e6b66b22f1f209861bee42 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:32,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:32,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T17:48:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:32,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124972988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:32,990 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in ee28898c42e6b66b22f1f209861bee42 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124973027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124973028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124973037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,122 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/6845d4e978ba4e8686aceb1badee7d36 2024-11-20T17:48:33,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/bc26d5ccb46446b89374b5d450e48865 is 50, key is test_row_0/B:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:33,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T17:48:33,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T17:48:33,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741957_1133 (size=12001) 2024-11-20T17:48:33,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T17:48:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124973337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124973338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124973347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T17:48:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T17:48:33,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/bc26d5ccb46446b89374b5d450e48865 2024-11-20T17:48:33,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/883050b3526b44a4b21be8e1adc1dafd is 50, key is test_row_0/C:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:33,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741958_1134 (size=12001) 2024-11-20T17:48:33,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/883050b3526b44a4b21be8e1adc1dafd 2024-11-20T17:48:33,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/6845d4e978ba4e8686aceb1badee7d36 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36 2024-11-20T17:48:33,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36, entries=200, sequenceid=52, filesize=38.6 K 2024-11-20T17:48:33,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/bc26d5ccb46446b89374b5d450e48865 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc26d5ccb46446b89374b5d450e48865 2024-11-20T17:48:33,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T17:48:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:33,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc26d5ccb46446b89374b5d450e48865, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T17:48:33,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/883050b3526b44a4b21be8e1adc1dafd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/883050b3526b44a4b21be8e1adc1dafd 2024-11-20T17:48:33,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/883050b3526b44a4b21be8e1adc1dafd, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T17:48:33,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee28898c42e6b66b22f1f209861bee42 in 1011ms, sequenceid=52, compaction requested=true 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:33,621 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:33,621 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:33,623 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:33,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:33,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:33,623 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:33,623 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,623 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,623 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=107.5 K 2024-11-20T17:48:33,623 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c2688ca1cc9b40f59af7678d02db277d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/226298bff2c44383885ab5c532390f4b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc26d5ccb46446b89374b5d450e48865] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.2 K 2024-11-20T17:48:33,623 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,624 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36] 2024-11-20T17:48:33,624 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c2688ca1cc9b40f59af7678d02db277d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732124910694 2024-11-20T17:48:33,624 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580ddb2af54141fd924b30a6fd5d3cfb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732124910694 2024-11-20T17:48:33,624 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 226298bff2c44383885ab5c532390f4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732124910785 2024-11-20T17:48:33,625 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09753210871e424f9076d5e329a68276, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732124910785 2024-11-20T17:48:33,625 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting bc26d5ccb46446b89374b5d450e48865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732124911964 2024-11-20T17:48:33,625 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6845d4e978ba4e8686aceb1badee7d36, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732124911958 2024-11-20T17:48:33,637 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:33,638 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:33,639 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/fb5788f9806b4e56b6b7c01860600ec1 is 50, key is test_row_0/B:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:33,645 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112093864e7fe9414338896735635846b60b_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:33,650 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112093864e7fe9414338896735635846b60b_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:33,650 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112093864e7fe9414338896735635846b60b_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:33,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741959_1135 (size=12104) 2024-11-20T17:48:33,669 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/fb5788f9806b4e56b6b7c01860600ec1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/fb5788f9806b4e56b6b7c01860600ec1 2024-11-20T17:48:33,683 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into fb5788f9806b4e56b6b7c01860600ec1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:33,683 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:33,684 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124913621; duration=0sec 2024-11-20T17:48:33,684 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:33,684 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:33,684 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:33,685 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:33,688 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:33,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741960_1136 (size=4469) 2024-11-20T17:48:33,688 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,688 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cf2ea60f46344714b4a47126ddfdd942, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/82feefe28c4e4ae29c487dd84ad01c28, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/883050b3526b44a4b21be8e1adc1dafd] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.2 K 2024-11-20T17:48:33,690 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2ea60f46344714b4a47126ddfdd942, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732124910694 2024-11-20T17:48:33,690 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#117 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:33,690 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 82feefe28c4e4ae29c487dd84ad01c28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732124910785 2024-11-20T17:48:33,691 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0dd7affbf2e54c358326f7c414d9aea9 is 175, key is test_row_0/A:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:33,691 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 883050b3526b44a4b21be8e1adc1dafd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732124911964 2024-11-20T17:48:33,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741961_1137 (size=31058) 2024-11-20T17:48:33,714 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:33,714 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/6cba74e13b3349adaef1882a144c37f2 is 50, key is test_row_0/C:col10/1732124911969/Put/seqid=0 2024-11-20T17:48:33,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741962_1138 (size=12104) 2024-11-20T17:48:33,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T17:48:33,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:33,756 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:48:33,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dd557dfe38e940ea89081fbe7466dfc7_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124912698/Put/seqid=0 2024-11-20T17:48:33,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741963_1139 (size=12154) 2024-11-20T17:48:33,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:33,802 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dd557dfe38e940ea89081fbe7466dfc7_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dd557dfe38e940ea89081fbe7466dfc7_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/7ab528b58aa448a4ab5e71e5ce97f6bd, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:33,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/7ab528b58aa448a4ab5e71e5ce97f6bd is 175, key is test_row_0/A:col10/1732124912698/Put/seqid=0 2024-11-20T17:48:33,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741964_1140 (size=30955) 2024-11-20T17:48:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:33,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124973869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124973882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124973884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T17:48:33,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124973987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124973987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:33,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124973997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,112 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0dd7affbf2e54c358326f7c414d9aea9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0dd7affbf2e54c358326f7c414d9aea9 2024-11-20T17:48:34,120 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into 0dd7affbf2e54c358326f7c414d9aea9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:34,120 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:34,120 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124913621; duration=0sec 2024-11-20T17:48:34,120 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:34,120 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:34,145 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/6cba74e13b3349adaef1882a144c37f2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/6cba74e13b3349adaef1882a144c37f2 2024-11-20T17:48:34,156 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 6cba74e13b3349adaef1882a144c37f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:34,156 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:34,156 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124913621; duration=0sec 2024-11-20T17:48:34,156 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:34,156 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:34,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124974197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124974198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124974207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,222 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/7ab528b58aa448a4ab5e71e5ce97f6bd 2024-11-20T17:48:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/6d10a257b9d74c8fb4aae198cb0b1675 is 50, key is test_row_0/B:col10/1732124912698/Put/seqid=0 2024-11-20T17:48:34,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741965_1141 (size=12001) 2024-11-20T17:48:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124974508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:34,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124974508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124974517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:34,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:34,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:34,658 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/6d10a257b9d74c8fb4aae198cb0b1675 2024-11-20T17:48:34,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/33566c5e9e6f46989105b3308d8f8dd0 is 50, key is test_row_0/C:col10/1732124912698/Put/seqid=0 2024-11-20T17:48:34,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741966_1142 (size=12001) 2024-11-20T17:48:34,695 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/33566c5e9e6f46989105b3308d8f8dd0 2024-11-20T17:48:34,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/7ab528b58aa448a4ab5e71e5ce97f6bd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd 2024-11-20T17:48:34,711 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd, entries=150, sequenceid=78, filesize=30.2 K 2024-11-20T17:48:34,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/6d10a257b9d74c8fb4aae198cb0b1675 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/6d10a257b9d74c8fb4aae198cb0b1675 2024-11-20T17:48:34,719 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/6d10a257b9d74c8fb4aae198cb0b1675, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T17:48:34,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/33566c5e9e6f46989105b3308d8f8dd0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/33566c5e9e6f46989105b3308d8f8dd0 2024-11-20T17:48:34,738 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/33566c5e9e6f46989105b3308d8f8dd0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T17:48:34,740 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for ee28898c42e6b66b22f1f209861bee42 in 983ms, sequenceid=78, compaction requested=false 2024-11-20T17:48:34,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:34,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:34,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T17:48:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T17:48:34,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T17:48:34,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-20T17:48:34,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9200 sec 2024-11-20T17:48:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T17:48:34,979 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:34,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T17:48:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T17:48:34,982 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:34,983 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:34,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:34,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:48:35,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207119b18d6c094273a3e5fc5d18a6ec53_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741967_1143 (size=14594) 2024-11-20T17:48:35,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:35,030 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207119b18d6c094273a3e5fc5d18a6ec53_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207119b18d6c094273a3e5fc5d18a6ec53_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:35,031 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/139210c22f3d4f15975d2dcd7591cb38, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:35,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/139210c22f3d4f15975d2dcd7591cb38 is 175, key is test_row_0/A:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124975040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124975042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124975044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124975046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124975046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741968_1144 (size=39549) 2024-11-20T17:48:35,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/139210c22f3d4f15975d2dcd7591cb38 2024-11-20T17:48:35,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/8277e362faa34b7d906b46d3e6e1dfe9 is 50, key is test_row_0/B:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741969_1145 (size=12001) 2024-11-20T17:48:35,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/8277e362faa34b7d906b46d3e6e1dfe9 2024-11-20T17:48:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T17:48:35,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/d8d118d4b3fc4b488f404ec20fce4a46 is 50, key is test_row_0/C:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741970_1146 (size=12001) 2024-11-20T17:48:35,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T17:48:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124975157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124975158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124975158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124975158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124975158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T17:48:35,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:35,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T17:48:35,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124975367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124975367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124975367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124975367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124975367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T17:48:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:35,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/d8d118d4b3fc4b488f404ec20fce4a46 2024-11-20T17:48:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/139210c22f3d4f15975d2dcd7591cb38 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38 2024-11-20T17:48:35,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38, entries=200, sequenceid=93, filesize=38.6 K 2024-11-20T17:48:35,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/8277e362faa34b7d906b46d3e6e1dfe9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8277e362faa34b7d906b46d3e6e1dfe9 2024-11-20T17:48:35,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8277e362faa34b7d906b46d3e6e1dfe9, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T17:48:35,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/d8d118d4b3fc4b488f404ec20fce4a46 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d8d118d4b3fc4b488f404ec20fce4a46 2024-11-20T17:48:35,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d8d118d4b3fc4b488f404ec20fce4a46, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T17:48:35,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for ee28898c42e6b66b22f1f209861bee42 in 541ms, sequenceid=93, compaction requested=true 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:35,532 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:35,532 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:35,533 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:35,533 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:35,533 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:35,533 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:35,533 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,533 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,533 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/fb5788f9806b4e56b6b7c01860600ec1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/6d10a257b9d74c8fb4aae198cb0b1675, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8277e362faa34b7d906b46d3e6e1dfe9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.3 K 2024-11-20T17:48:35,533 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0dd7affbf2e54c358326f7c414d9aea9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=99.2 K 2024-11-20T17:48:35,533 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,534 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0dd7affbf2e54c358326f7c414d9aea9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38] 2024-11-20T17:48:35,534 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting fb5788f9806b4e56b6b7c01860600ec1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732124911964 2024-11-20T17:48:35,534 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd7affbf2e54c358326f7c414d9aea9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732124911964 2024-11-20T17:48:35,534 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d10a257b9d74c8fb4aae198cb0b1675, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732124912681 2024-11-20T17:48:35,534 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab528b58aa448a4ab5e71e5ce97f6bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732124912681 2024-11-20T17:48:35,535 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8277e362faa34b7d906b46d3e6e1dfe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732124913864 2024-11-20T17:48:35,535 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 139210c22f3d4f15975d2dcd7591cb38, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732124913864 2024-11-20T17:48:35,545 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:35,546 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#125 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:35,546 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/aa90b045856e42c981cbb8a9f155f555 is 50, key is test_row_0/B:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,561 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b3a03cfa25ea47549e96a37b60dd1842_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:35,564 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b3a03cfa25ea47549e96a37b60dd1842_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:35,564 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b3a03cfa25ea47549e96a37b60dd1842_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:35,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741971_1147 (size=12207) 2024-11-20T17:48:35,577 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/aa90b045856e42c981cbb8a9f155f555 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa90b045856e42c981cbb8a9f155f555 2024-11-20T17:48:35,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741972_1148 (size=4469) 2024-11-20T17:48:35,584 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into aa90b045856e42c981cbb8a9f155f555(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:35,584 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:35,584 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124915532; duration=0sec 2024-11-20T17:48:35,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:35,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:35,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:35,586 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:35,586 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:35,586 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,586 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/6cba74e13b3349adaef1882a144c37f2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/33566c5e9e6f46989105b3308d8f8dd0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d8d118d4b3fc4b488f404ec20fce4a46] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.3 K 2024-11-20T17:48:35,587 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cba74e13b3349adaef1882a144c37f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732124911964 2024-11-20T17:48:35,588 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 33566c5e9e6f46989105b3308d8f8dd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732124912681 2024-11-20T17:48:35,588 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d118d4b3fc4b488f404ec20fce4a46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732124913864 2024-11-20T17:48:35,596 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:35,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T17:48:35,597 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/4e544b8d93144013af0d4e31ec4e1ff1 is 50, key is test_row_0/C:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:35,597 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:48:35,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741973_1149 (size=12207) 2024-11-20T17:48:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a0cfd6b3f78b42f2baec93bd83d31b1d_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124915043/Put/seqid=0 2024-11-20T17:48:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T17:48:35,608 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/4e544b8d93144013af0d4e31ec4e1ff1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/4e544b8d93144013af0d4e31ec4e1ff1 2024-11-20T17:48:35,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741974_1150 (size=12154) 2024-11-20T17:48:35,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:35,617 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 4e544b8d93144013af0d4e31ec4e1ff1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:35,617 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:35,617 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124915532; duration=0sec 2024-11-20T17:48:35,617 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:35,617 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:35,619 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a0cfd6b3f78b42f2baec93bd83d31b1d_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a0cfd6b3f78b42f2baec93bd83d31b1d_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/bd115ce7099b430f9a8e925096ecbb21, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/bd115ce7099b430f9a8e925096ecbb21 is 175, key is test_row_0/A:col10/1732124915043/Put/seqid=0 2024-11-20T17:48:35,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741975_1151 (size=30955) 2024-11-20T17:48:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:35,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124975685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124975686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124975687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124975688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124975688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124975797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124975797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124975797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124975797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124975797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:35,979 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#126 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:35,980 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0f2dab3fb7744880ac78586af1b58442 is 175, key is test_row_0/A:col10/1732124913864/Put/seqid=0 2024-11-20T17:48:35,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741976_1152 (size=31161) 2024-11-20T17:48:36,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124976007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124976007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124976007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124976007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124976007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,026 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/bd115ce7099b430f9a8e925096ecbb21 2024-11-20T17:48:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/ece1063b31b741c4b386b9f7a336ee49 is 50, key is test_row_0/B:col10/1732124915043/Put/seqid=0 2024-11-20T17:48:36,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741977_1153 (size=12001) 2024-11-20T17:48:36,044 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/ece1063b31b741c4b386b9f7a336ee49 2024-11-20T17:48:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5a162ce09f2246c286a3a47940fd15d3 is 50, key is test_row_0/C:col10/1732124915043/Put/seqid=0 2024-11-20T17:48:36,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741978_1154 (size=12001) 2024-11-20T17:48:36,079 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5a162ce09f2246c286a3a47940fd15d3 2024-11-20T17:48:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/bd115ce7099b430f9a8e925096ecbb21 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21 2024-11-20T17:48:36,096 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21, entries=150, sequenceid=118, filesize=30.2 K 2024-11-20T17:48:36,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/ece1063b31b741c4b386b9f7a336ee49 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/ece1063b31b741c4b386b9f7a336ee49 2024-11-20T17:48:36,109 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/ece1063b31b741c4b386b9f7a336ee49, entries=150, sequenceid=118, filesize=11.7 K 2024-11-20T17:48:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5a162ce09f2246c286a3a47940fd15d3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5a162ce09f2246c286a3a47940fd15d3 2024-11-20T17:48:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T17:48:36,120 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5a162ce09f2246c286a3a47940fd15d3, entries=150, sequenceid=118, filesize=11.7 K 2024-11-20T17:48:36,123 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for ee28898c42e6b66b22f1f209861bee42 in 525ms, sequenceid=118, compaction requested=false 2024-11-20T17:48:36,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:36,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:36,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T17:48:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T17:48:36,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T17:48:36,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-20T17:48:36,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1480 sec 2024-11-20T17:48:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:36,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T17:48:36,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124976333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124976333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124976334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124976334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124976335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209b71ef305d834fb38261e0b240248d63_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741979_1155 (size=12254) 2024-11-20T17:48:36,359 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:36,363 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209b71ef305d834fb38261e0b240248d63_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209b71ef305d834fb38261e0b240248d63_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:36,364 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0477a678cf64486d80ab2293b29b587c, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:36,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0477a678cf64486d80ab2293b29b587c is 175, key is test_row_0/A:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741980_1156 (size=31055) 2024-11-20T17:48:36,390 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0f2dab3fb7744880ac78586af1b58442 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0f2dab3fb7744880ac78586af1b58442 2024-11-20T17:48:36,398 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into 0f2dab3fb7744880ac78586af1b58442(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:36,398 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:36,398 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124915532; duration=0sec 2024-11-20T17:48:36,398 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:36,399 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124976437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124976437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124976447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124976447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124976447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124976647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124976648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124976657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124976657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124976657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,769 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0477a678cf64486d80ab2293b29b587c 2024-11-20T17:48:36,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/8d2d5c2efcb64d548460d0f59e2ce97a is 50, key is test_row_0/B:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741981_1157 (size=12101) 2024-11-20T17:48:36,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/8d2d5c2efcb64d548460d0f59e2ce97a 2024-11-20T17:48:36,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/411511de4aac4fb199a0fe02758d1598 is 50, key is test_row_0/C:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741982_1158 (size=12101) 2024-11-20T17:48:36,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/411511de4aac4fb199a0fe02758d1598 2024-11-20T17:48:36,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/0477a678cf64486d80ab2293b29b587c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c 2024-11-20T17:48:36,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c, entries=150, sequenceid=133, filesize=30.3 K 2024-11-20T17:48:36,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/8d2d5c2efcb64d548460d0f59e2ce97a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8d2d5c2efcb64d548460d0f59e2ce97a 2024-11-20T17:48:36,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8d2d5c2efcb64d548460d0f59e2ce97a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-20T17:48:36,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/411511de4aac4fb199a0fe02758d1598 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/411511de4aac4fb199a0fe02758d1598 2024-11-20T17:48:36,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/411511de4aac4fb199a0fe02758d1598, entries=150, sequenceid=133, filesize=11.8 K 2024-11-20T17:48:36,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for ee28898c42e6b66b22f1f209861bee42 in 494ms, sequenceid=133, compaction requested=true 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:36,812 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:36,812 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:36,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:36,814 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:36,814 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:36,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:36,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:36,814 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:36,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0f2dab3fb7744880ac78586af1b58442, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=91.0 K 2024-11-20T17:48:36,814 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa90b045856e42c981cbb8a9f155f555, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/ece1063b31b741c4b386b9f7a336ee49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8d2d5c2efcb64d548460d0f59e2ce97a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.5 K 2024-11-20T17:48:36,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:36,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0f2dab3fb7744880ac78586af1b58442, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c] 2024-11-20T17:48:36,814 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting aa90b045856e42c981cbb8a9f155f555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732124913864 2024-11-20T17:48:36,815 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2dab3fb7744880ac78586af1b58442, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732124913864 2024-11-20T17:48:36,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ece1063b31b741c4b386b9f7a336ee49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732124915035 2024-11-20T17:48:36,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2d5c2efcb64d548460d0f59e2ce97a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732124915682 2024-11-20T17:48:36,816 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd115ce7099b430f9a8e925096ecbb21, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732124915035 2024-11-20T17:48:36,816 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0477a678cf64486d80ab2293b29b587c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732124915682 2024-11-20T17:48:36,823 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:36,824 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/2bed0d05e5cf4000bf10b7ad34301bf4 is 50, key is test_row_0/B:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,826 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:36,835 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411200c862f805bb34577b5f3d29f2ff77e33_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:36,838 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411200c862f805bb34577b5f3d29f2ff77e33_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:36,838 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200c862f805bb34577b5f3d29f2ff77e33_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741983_1159 (size=12409) 2024-11-20T17:48:36,846 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/2bed0d05e5cf4000bf10b7ad34301bf4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/2bed0d05e5cf4000bf10b7ad34301bf4 2024-11-20T17:48:36,852 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into 2bed0d05e5cf4000bf10b7ad34301bf4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:36,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:36,852 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124916812; duration=0sec 2024-11-20T17:48:36,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:36,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:36,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:36,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741984_1160 (size=4469) 2024-11-20T17:48:36,854 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#135 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:36,855 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/c0c09664bd6d4c7aa91a73569f1ad143 is 175, key is test_row_0/A:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,857 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:36,857 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:36,857 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:36,857 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/4e544b8d93144013af0d4e31ec4e1ff1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5a162ce09f2246c286a3a47940fd15d3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/411511de4aac4fb199a0fe02758d1598] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.5 K 2024-11-20T17:48:36,858 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e544b8d93144013af0d4e31ec4e1ff1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732124913864 2024-11-20T17:48:36,858 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a162ce09f2246c286a3a47940fd15d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732124915035 2024-11-20T17:48:36,858 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 411511de4aac4fb199a0fe02758d1598, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732124915682 2024-11-20T17:48:36,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741985_1161 (size=31363) 2024-11-20T17:48:36,870 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:36,870 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/35c6793e2c9b41a6bf0cc253ed86ac7e is 50, key is test_row_0/C:col10/1732124916317/Put/seqid=0 2024-11-20T17:48:36,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741986_1162 (size=12409) 2024-11-20T17:48:36,895 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/35c6793e2c9b41a6bf0cc253ed86ac7e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/35c6793e2c9b41a6bf0cc253ed86ac7e 2024-11-20T17:48:36,902 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 35c6793e2c9b41a6bf0cc253ed86ac7e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:36,902 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:36,903 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124916812; duration=0sec 2024-11-20T17:48:36,903 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:36,903 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:36,960 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:36,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:48:36,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d01f59f7503940598bcc2691b25d6a01_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124916957/Put/seqid=0 2024-11-20T17:48:36,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124976973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124976974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124976975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124976975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124976976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:36,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741987_1163 (size=14794) 2024-11-20T17:48:37,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:37,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d01f59f7503940598bcc2691b25d6a01_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d01f59f7503940598bcc2691b25d6a01_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:37,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/ee8a223a733f4f619b27e92896c75016, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:37,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/ee8a223a733f4f619b27e92896c75016 is 175, key is test_row_0/A:col10/1732124916957/Put/seqid=0 2024-11-20T17:48:37,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741988_1164 (size=39749) 2024-11-20T17:48:37,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/ee8a223a733f4f619b27e92896c75016 2024-11-20T17:48:37,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/0547fbf618f74915b6ebde8be9682288 is 50, key is test_row_0/B:col10/1732124916957/Put/seqid=0 2024-11-20T17:48:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741989_1165 (size=12151) 2024-11-20T17:48:37,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/0547fbf618f74915b6ebde8be9682288 2024-11-20T17:48:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124977087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124977087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124977087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124977088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124977088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/f9f6cd3d4bf04a28a01bde3cb1cb3714 is 50, key is test_row_0/C:col10/1732124916957/Put/seqid=0 2024-11-20T17:48:37,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741990_1166 (size=12151) 2024-11-20T17:48:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T17:48:37,127 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:37,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T17:48:37,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T17:48:37,131 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:37,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T17:48:37,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:37,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:37,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124977298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124977298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124977298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124977298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124977298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,316 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/c0c09664bd6d4c7aa91a73569f1ad143 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0c09664bd6d4c7aa91a73569f1ad143 2024-11-20T17:48:37,323 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into c0c09664bd6d4c7aa91a73569f1ad143(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:37,324 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:37,324 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124916812; duration=0sec 2024-11-20T17:48:37,324 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:37,324 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T17:48:37,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,467 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39775 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=613581fcdc40,44957,1732124872893, table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T17:48:37,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/f9f6cd3d4bf04a28a01bde3cb1cb3714 2024-11-20T17:48:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/ee8a223a733f4f619b27e92896c75016 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016 2024-11-20T17:48:37,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016, entries=200, sequenceid=160, filesize=38.8 K 2024-11-20T17:48:37,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/0547fbf618f74915b6ebde8be9682288 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0547fbf618f74915b6ebde8be9682288 2024-11-20T17:48:37,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0547fbf618f74915b6ebde8be9682288, entries=150, sequenceid=160, filesize=11.9 K 2024-11-20T17:48:37,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/f9f6cd3d4bf04a28a01bde3cb1cb3714 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/f9f6cd3d4bf04a28a01bde3cb1cb3714 2024-11-20T17:48:37,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/f9f6cd3d4bf04a28a01bde3cb1cb3714, entries=150, sequenceid=160, filesize=11.9 K 2024-11-20T17:48:37,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ee28898c42e6b66b22f1f209861bee42 in 584ms, sequenceid=160, compaction requested=false 2024-11-20T17:48:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:37,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:48:37,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,638 INFO [master/613581fcdc40:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T17:48:37,638 INFO [master/613581fcdc40:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T17:48:37,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ea7def61aa4643828e8d6dda39706fb3_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124977640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124977640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124977640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124977641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124977647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741991_1167 (size=14794) 2024-11-20T17:48:37,656 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:37,661 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ea7def61aa4643828e8d6dda39706fb3_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea7def61aa4643828e8d6dda39706fb3_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:37,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/00742f98d7944e18b967e0bbc466c329, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:37,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/00742f98d7944e18b967e0bbc466c329 is 175, key is test_row_0/A:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:37,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741992_1168 (size=39749) 2024-11-20T17:48:37,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/00742f98d7944e18b967e0bbc466c329 2024-11-20T17:48:37,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/18d4f5f18ef5451d99ab9b248155208c is 50, key is test_row_0/B:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741993_1169 (size=12151) 2024-11-20T17:48:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T17:48:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124977757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124977757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124977758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124977758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124977758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:37,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:37,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:37,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124977967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124977968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124977968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124977968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124977968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/18d4f5f18ef5451d99ab9b248155208c 2024-11-20T17:48:38,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/01849e6141174b2ca0e9557e7546753c is 50, key is test_row_0/C:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:38,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741994_1170 (size=12151) 2024-11-20T17:48:38,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T17:48:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124978277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124978277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124978278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124978278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124978278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:38,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:38,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/01849e6141174b2ca0e9557e7546753c 2024-11-20T17:48:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/00742f98d7944e18b967e0bbc466c329 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329 2024-11-20T17:48:38,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329, entries=200, sequenceid=174, filesize=38.8 K 2024-11-20T17:48:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/18d4f5f18ef5451d99ab9b248155208c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/18d4f5f18ef5451d99ab9b248155208c 2024-11-20T17:48:38,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/18d4f5f18ef5451d99ab9b248155208c, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T17:48:38,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/01849e6141174b2ca0e9557e7546753c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/01849e6141174b2ca0e9557e7546753c 2024-11-20T17:48:38,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/01849e6141174b2ca0e9557e7546753c, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T17:48:38,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ee28898c42e6b66b22f1f209861bee42 in 926ms, sequenceid=174, compaction requested=true 2024-11-20T17:48:38,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:38,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:38,537 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:38,537 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:38,538 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:38,538 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:38,538 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:38,538 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:38,538 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,538 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,538 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/2bed0d05e5cf4000bf10b7ad34301bf4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0547fbf618f74915b6ebde8be9682288, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/18d4f5f18ef5451d99ab9b248155208c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.9 K 2024-11-20T17:48:38,538 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0c09664bd6d4c7aa91a73569f1ad143, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=108.3 K 2024-11-20T17:48:38,538 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,538 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0c09664bd6d4c7aa91a73569f1ad143, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329] 2024-11-20T17:48:38,539 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bed0d05e5cf4000bf10b7ad34301bf4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732124915682 2024-11-20T17:48:38,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T17:48:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,540 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:48:38,540 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0547fbf618f74915b6ebde8be9682288, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732124916332 2024-11-20T17:48:38,540 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c09664bd6d4c7aa91a73569f1ad143, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732124915682 2024-11-20T17:48:38,541 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d4f5f18ef5451d99ab9b248155208c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732124916973 2024-11-20T17:48:38,541 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8a223a733f4f619b27e92896c75016, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732124916331 2024-11-20T17:48:38,542 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00742f98d7944e18b967e0bbc466c329, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732124916973 2024-11-20T17:48:38,551 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:38,551 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/b8a80dfb44b74a3197da810cd82e5fe8 is 50, key is test_row_0/B:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:38,553 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b2218fcf221c4c7fbb1b261d70f7e1a9_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124917638/Put/seqid=0 2024-11-20T17:48:38,561 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411203168fd730be94652a96f5a9b7d509921_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:38,563 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411203168fd730be94652a96f5a9b7d509921_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:38,563 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203168fd730be94652a96f5a9b7d509921_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:38,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741995_1171 (size=12561) 2024-11-20T17:48:38,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741997_1173 (size=4469) 2024-11-20T17:48:38,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741996_1172 (size=12304) 2024-11-20T17:48:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/b8a80dfb44b74a3197da810cd82e5fe8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/b8a80dfb44b74a3197da810cd82e5fe8 2024-11-20T17:48:38,589 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into b8a80dfb44b74a3197da810cd82e5fe8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:38,589 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:38,589 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124918537; duration=0sec 2024-11-20T17:48:38,590 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:38,590 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:38,590 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:38,593 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b2218fcf221c4c7fbb1b261d70f7e1a9_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b2218fcf221c4c7fbb1b261d70f7e1a9_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:38,594 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:38,594 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:38,594 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,594 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/35c6793e2c9b41a6bf0cc253ed86ac7e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/f9f6cd3d4bf04a28a01bde3cb1cb3714, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/01849e6141174b2ca0e9557e7546753c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=35.9 K 2024-11-20T17:48:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/640cd2e9e28e4be498246481c6020ff6, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:38,595 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c6793e2c9b41a6bf0cc253ed86ac7e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732124915682 2024-11-20T17:48:38,595 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f6cd3d4bf04a28a01bde3cb1cb3714, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732124916332 2024-11-20T17:48:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/640cd2e9e28e4be498246481c6020ff6 is 175, key is test_row_0/A:col10/1732124917638/Put/seqid=0 2024-11-20T17:48:38,596 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 01849e6141174b2ca0e9557e7546753c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732124916973 2024-11-20T17:48:38,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741998_1174 (size=31105) 2024-11-20T17:48:38,601 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/640cd2e9e28e4be498246481c6020ff6 2024-11-20T17:48:38,606 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:38,607 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/98e71eb3b19f4ef49cc505c981297fbc is 50, key is test_row_0/C:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:38,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741999_1175 (size=12561) 2024-11-20T17:48:38,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/bc06b2c9911c4c289b83bbbf90a40d7e is 50, key is test_row_0/B:col10/1732124917638/Put/seqid=0 2024-11-20T17:48:38,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742000_1176 (size=12151) 2024-11-20T17:48:38,626 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/bc06b2c9911c4c289b83bbbf90a40d7e 2024-11-20T17:48:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/397234fd99334732aecff7c35a9acbca is 50, key is test_row_0/C:col10/1732124917638/Put/seqid=0 2024-11-20T17:48:38,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742001_1177 (size=12151) 2024-11-20T17:48:38,647 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/397234fd99334732aecff7c35a9acbca 2024-11-20T17:48:38,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/640cd2e9e28e4be498246481c6020ff6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6 2024-11-20T17:48:38,663 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6, entries=150, sequenceid=197, filesize=30.4 K 2024-11-20T17:48:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/bc06b2c9911c4c289b83bbbf90a40d7e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc06b2c9911c4c289b83bbbf90a40d7e 2024-11-20T17:48:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,669 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc06b2c9911c4c289b83bbbf90a40d7e, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T17:48:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/397234fd99334732aecff7c35a9acbca as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/397234fd99334732aecff7c35a9acbca 2024-11-20T17:48:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,678 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/397234fd99334732aecff7c35a9acbca, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T17:48:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,680 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for ee28898c42e6b66b22f1f209861bee42 in 140ms, sequenceid=197, compaction requested=false 2024-11-20T17:48:38,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:38,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T17:48:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T17:48:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T17:48:38,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-11-20T17:48:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5570 sec 2024-11-20T17:48:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:38,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:38,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:48:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de43abbc75cf47d898e913510400c692_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742002_1178 (size=14794) 2024-11-20T17:48:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:38,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124978919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124978921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124978921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124978921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124978922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:38,976 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:38,977 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/c0eddafe9c2b443087f0e89835cfe8c8 is 175, key is test_row_0/A:col10/1732124916974/Put/seqid=0 2024-11-20T17:48:38,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742003_1179 (size=31515) 2024-11-20T17:48:39,021 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/98e71eb3b19f4ef49cc505c981297fbc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/98e71eb3b19f4ef49cc505c981297fbc 2024-11-20T17:48:39,026 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 98e71eb3b19f4ef49cc505c981297fbc(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:39,026 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:39,026 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124918537; duration=0sec 2024-11-20T17:48:39,026 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:39,026 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:39,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124979027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124979037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124979037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124979037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124979037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124979237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124979247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124979247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124979247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124979247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T17:48:39,287 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:39,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T17:48:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T17:48:39,290 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:39,290 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:39,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:39,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:39,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de43abbc75cf47d898e913510400c692_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de43abbc75cf47d898e913510400c692_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:39,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/250d5d15cb814792b7773bdcd45617a0, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:39,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/250d5d15cb814792b7773bdcd45617a0 is 175, key is test_row_0/A:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742004_1180 (size=39749) 2024-11-20T17:48:39,386 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/c0eddafe9c2b443087f0e89835cfe8c8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0eddafe9c2b443087f0e89835cfe8c8 2024-11-20T17:48:39,392 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into c0eddafe9c2b443087f0e89835cfe8c8(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:39,392 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:39,392 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124918536; duration=0sec 2024-11-20T17:48:39,393 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:39,393 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T17:48:39,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T17:48:39,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124979547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124979557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124979557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124979558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124979558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:39,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T17:48:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T17:48:39,725 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/250d5d15cb814792b7773bdcd45617a0 2024-11-20T17:48:39,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/071058472ec94349a938320b81cba457 is 50, key is test_row_0/B:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:39,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T17:48:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:39,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742005_1181 (size=12151) 2024-11-20T17:48:39,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/071058472ec94349a938320b81cba457 2024-11-20T17:48:39,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/2fe7ecc7a9524a00876faeb3fedd4dd2 is 50, key is test_row_0/C:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:39,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742006_1182 (size=12151) 2024-11-20T17:48:39,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/2fe7ecc7a9524a00876faeb3fedd4dd2 2024-11-20T17:48:39,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/250d5d15cb814792b7773bdcd45617a0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0 2024-11-20T17:48:39,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0, entries=200, sequenceid=209, filesize=38.8 K 2024-11-20T17:48:39,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/071058472ec94349a938320b81cba457 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/071058472ec94349a938320b81cba457 2024-11-20T17:48:39,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/071058472ec94349a938320b81cba457, entries=150, sequenceid=209, filesize=11.9 K 2024-11-20T17:48:39,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/2fe7ecc7a9524a00876faeb3fedd4dd2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2fe7ecc7a9524a00876faeb3fedd4dd2 2024-11-20T17:48:39,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2fe7ecc7a9524a00876faeb3fedd4dd2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-20T17:48:39,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee28898c42e6b66b22f1f209861bee42 in 943ms, sequenceid=209, compaction requested=true 2024-11-20T17:48:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:39,798 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:39,798 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:39,799 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,799 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,799 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0eddafe9c2b443087f0e89835cfe8c8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=100.0 K 2024-11-20T17:48:39,799 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/b8a80dfb44b74a3197da810cd82e5fe8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc06b2c9911c4c289b83bbbf90a40d7e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/071058472ec94349a938320b81cba457] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.0 K 2024-11-20T17:48:39,799 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0eddafe9c2b443087f0e89835cfe8c8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0] 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b8a80dfb44b74a3197da810cd82e5fe8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732124916973 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0eddafe9c2b443087f0e89835cfe8c8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732124916973 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting bc06b2c9911c4c289b83bbbf90a40d7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732124917633 2024-11-20T17:48:39,799 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 640cd2e9e28e4be498246481c6020ff6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732124917633 2024-11-20T17:48:39,800 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 071058472ec94349a938320b81cba457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732124918847 2024-11-20T17:48:39,800 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 250d5d15cb814792b7773bdcd45617a0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732124918847 2024-11-20T17:48:39,810 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:39,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120079f7851be864e7dbd47724df3e30fbb_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:39,814 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:39,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/711226785f3342519a38ab1cea3f1a34 is 50, key is test_row_0/B:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:39,817 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120079f7851be864e7dbd47724df3e30fbb_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:39,817 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120079f7851be864e7dbd47724df3e30fbb_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:39,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742007_1183 (size=12663) 2024-11-20T17:48:39,843 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/711226785f3342519a38ab1cea3f1a34 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/711226785f3342519a38ab1cea3f1a34 2024-11-20T17:48:39,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742008_1184 (size=4469) 2024-11-20T17:48:39,849 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#152 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:39,849 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into 711226785f3342519a38ab1cea3f1a34(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:39,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:39,849 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/c8841e97b9e24273a11c926253cb3ea0 is 175, key is test_row_0/A:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:39,849 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124919798; duration=0sec 2024-11-20T17:48:39,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:39,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:39,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:39,853 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:39,853 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:39,853 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,853 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/98e71eb3b19f4ef49cc505c981297fbc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/397234fd99334732aecff7c35a9acbca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2fe7ecc7a9524a00876faeb3fedd4dd2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.0 K 2024-11-20T17:48:39,854 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e71eb3b19f4ef49cc505c981297fbc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732124916973 2024-11-20T17:48:39,854 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 397234fd99334732aecff7c35a9acbca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732124917633 2024-11-20T17:48:39,854 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fe7ecc7a9524a00876faeb3fedd4dd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732124918847 2024-11-20T17:48:39,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742009_1185 (size=31617) 2024-11-20T17:48:39,866 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:39,867 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/37652b68b44740078d87557ab0a08328 is 50, key is test_row_0/C:col10/1732124918847/Put/seqid=0 2024-11-20T17:48:39,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742010_1186 (size=12663) 2024-11-20T17:48:39,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T17:48:39,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:39,907 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:48:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T17:48:39,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208e0bd79a8eca4198a09a13758dafa0b3_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124918919/Put/seqid=0 2024-11-20T17:48:39,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742011_1187 (size=12304) 2024-11-20T17:48:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:40,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124980075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124980075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124980077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124980077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124980079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124980187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124980187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124980187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124980187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124980197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,270 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/c8841e97b9e24273a11c926253cb3ea0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c8841e97b9e24273a11c926253cb3ea0 2024-11-20T17:48:40,276 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/37652b68b44740078d87557ab0a08328 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/37652b68b44740078d87557ab0a08328 2024-11-20T17:48:40,277 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into c8841e97b9e24273a11c926253cb3ea0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:40,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:40,277 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124919798; duration=0sec 2024-11-20T17:48:40,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:40,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:40,282 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 37652b68b44740078d87557ab0a08328(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:40,282 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:40,282 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124919798; duration=0sec 2024-11-20T17:48:40,282 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:40,282 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:40,360 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208e0bd79a8eca4198a09a13758dafa0b3_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208e0bd79a8eca4198a09a13758dafa0b3_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:40,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/e42746dcd6a145b2b1fdc06719b30bbc, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:40,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/e42746dcd6a145b2b1fdc06719b30bbc is 175, key is test_row_0/A:col10/1732124918919/Put/seqid=0 2024-11-20T17:48:40,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742012_1188 (size=31105) 2024-11-20T17:48:40,382 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/e42746dcd6a145b2b1fdc06719b30bbc 2024-11-20T17:48:40,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/302592a2ce84477194a8e613d3611ddd is 50, key is test_row_0/B:col10/1732124918919/Put/seqid=0 2024-11-20T17:48:40,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124980398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124980398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124980398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124980398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742013_1189 (size=12151) 2024-11-20T17:48:40,410 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/302592a2ce84477194a8e613d3611ddd 2024-11-20T17:48:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124980407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/1dacf0c2926644bf90352a6acc3a1c7a is 50, key is test_row_0/C:col10/1732124918919/Put/seqid=0 2024-11-20T17:48:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T17:48:40,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742014_1190 (size=12151) 2024-11-20T17:48:40,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124980707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124980707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124980708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124980708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124980717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:40,832 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/1dacf0c2926644bf90352a6acc3a1c7a 2024-11-20T17:48:40,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/e42746dcd6a145b2b1fdc06719b30bbc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc 2024-11-20T17:48:40,841 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc, entries=150, sequenceid=237, filesize=30.4 K 2024-11-20T17:48:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/302592a2ce84477194a8e613d3611ddd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/302592a2ce84477194a8e613d3611ddd 2024-11-20T17:48:40,847 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/302592a2ce84477194a8e613d3611ddd, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T17:48:40,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/1dacf0c2926644bf90352a6acc3a1c7a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/1dacf0c2926644bf90352a6acc3a1c7a 2024-11-20T17:48:40,855 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/1dacf0c2926644bf90352a6acc3a1c7a, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T17:48:40,856 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee28898c42e6b66b22f1f209861bee42 in 949ms, sequenceid=237, compaction requested=false 2024-11-20T17:48:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T17:48:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T17:48:40,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T17:48:40,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-20T17:48:40,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5710 sec 2024-11-20T17:48:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:41,220 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:41,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:48:41,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a3ee1453a0fd47f3bd251989ef1a5417_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:41,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742015_1191 (size=17284) 2024-11-20T17:48:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124981253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124981254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124981254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124981256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124981256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124981367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124981367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124981367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124981377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124981378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T17:48:41,437 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:41,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-20T17:48:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:41,439 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:41,440 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:41,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124981577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124981587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124981587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124981587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124981587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:41,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,639 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:41,643 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a3ee1453a0fd47f3bd251989ef1a5417_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a3ee1453a0fd47f3bd251989ef1a5417_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:41,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/324d50479dde4dea838fa63c73581be3, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:41,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/324d50479dde4dea838fa63c73581be3 is 175, key is test_row_0/A:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:41,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742016_1192 (size=48389) 2024-11-20T17:48:41,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:41,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124981887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:41,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:41,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124981898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124981897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124981898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124981898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:42,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/324d50479dde4dea838fa63c73581be3 2024-11-20T17:48:42,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/0a1af0d3c8514aab88c45ae58c5893d5 is 50, key is test_row_0/B:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:42,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742017_1193 (size=12151) 2024-11-20T17:48:42,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/0a1af0d3c8514aab88c45ae58c5893d5 2024-11-20T17:48:42,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/cb8180ea0277465aa4c8bab34d05e568 is 50, key is test_row_0/C:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:42,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742018_1194 (size=12151) 2024-11-20T17:48:42,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:42,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:42,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124982417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:42,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/cb8180ea0277465aa4c8bab34d05e568 2024-11-20T17:48:42,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/324d50479dde4dea838fa63c73581be3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3 2024-11-20T17:48:42,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3, entries=250, sequenceid=252, filesize=47.3 K 2024-11-20T17:48:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/0a1af0d3c8514aab88c45ae58c5893d5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0a1af0d3c8514aab88c45ae58c5893d5 2024-11-20T17:48:42,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0a1af0d3c8514aab88c45ae58c5893d5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-20T17:48:42,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/cb8180ea0277465aa4c8bab34d05e568 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cb8180ea0277465aa4c8bab34d05e568 2024-11-20T17:48:42,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cb8180ea0277465aa4c8bab34d05e568, entries=150, sequenceid=252, filesize=11.9 K 2024-11-20T17:48:42,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ee28898c42e6b66b22f1f209861bee42 in 1296ms, sequenceid=252, compaction requested=true 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:42,517 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:42,517 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:42,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:42,518 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:42,518 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:42,518 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:42,518 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:42,518 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,518 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,519 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/711226785f3342519a38ab1cea3f1a34, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/302592a2ce84477194a8e613d3611ddd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0a1af0d3c8514aab88c45ae58c5893d5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.1 K 2024-11-20T17:48:42,519 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c8841e97b9e24273a11c926253cb3ea0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=108.5 K 2024-11-20T17:48:42,519 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,519 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c8841e97b9e24273a11c926253cb3ea0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3] 2024-11-20T17:48:42,519 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 711226785f3342519a38ab1cea3f1a34, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732124918847 2024-11-20T17:48:42,519 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8841e97b9e24273a11c926253cb3ea0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732124918847 2024-11-20T17:48:42,519 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42746dcd6a145b2b1fdc06719b30bbc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732124918901 2024-11-20T17:48:42,519 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 302592a2ce84477194a8e613d3611ddd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732124918901 2024-11-20T17:48:42,520 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1af0d3c8514aab88c45ae58c5893d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124920075 2024-11-20T17:48:42,520 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 324d50479dde4dea838fa63c73581be3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124920075 2024-11-20T17:48:42,550 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:42,550 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/10249778768740dea4797ef0a1e7197f is 50, key is test_row_0/B:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:42,558 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:42,564 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112033d9607463be48cfa4a1d6f14a352030_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:42,567 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112033d9607463be48cfa4a1d6f14a352030_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:42,567 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112033d9607463be48cfa4a1d6f14a352030_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:42,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742019_1195 (size=12765) 2024-11-20T17:48:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:42,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742020_1196 (size=4469) 2024-11-20T17:48:42,591 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#162 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:42,592 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/e7a176a1f08a4ed58bd463c6aca38993 is 175, key is test_row_0/A:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742021_1197 (size=31719) 2024-11-20T17:48:42,613 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/e7a176a1f08a4ed58bd463c6aca38993 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e7a176a1f08a4ed58bd463c6aca38993 2024-11-20T17:48:42,623 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into e7a176a1f08a4ed58bd463c6aca38993(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:42,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:42,623 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124922517; duration=0sec 2024-11-20T17:48:42,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:42,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:42,623 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:42,625 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:42,625 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:42,625 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,625 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/37652b68b44740078d87557ab0a08328, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/1dacf0c2926644bf90352a6acc3a1c7a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cb8180ea0277465aa4c8bab34d05e568] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.1 K 2024-11-20T17:48:42,626 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37652b68b44740078d87557ab0a08328, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732124918847 2024-11-20T17:48:42,626 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dacf0c2926644bf90352a6acc3a1c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732124918901 2024-11-20T17:48:42,627 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb8180ea0277465aa4c8bab34d05e568, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124920075 2024-11-20T17:48:42,643 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:42,644 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/17b950182bca4275b63ab524ba5fdb60 is 50, key is test_row_0/C:col10/1732124921219/Put/seqid=0 2024-11-20T17:48:42,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742022_1198 (size=12765) 2024-11-20T17:48:42,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T17:48:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:42,671 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:48:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d3434ff01726422d95ac7db95a00defc_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124921252/Put/seqid=0 2024-11-20T17:48:42,691 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/17b950182bca4275b63ab524ba5fdb60 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/17b950182bca4275b63ab524ba5fdb60 2024-11-20T17:48:42,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742023_1199 (size=12454) 2024-11-20T17:48:42,698 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 17b950182bca4275b63ab524ba5fdb60(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:42,698 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:42,698 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124922517; duration=0sec 2024-11-20T17:48:42,698 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:42,698 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:42,983 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/10249778768740dea4797ef0a1e7197f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/10249778768740dea4797ef0a1e7197f 2024-11-20T17:48:42,989 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into 10249778768740dea4797ef0a1e7197f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:42,990 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:42,990 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124922517; duration=0sec 2024-11-20T17:48:42,990 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:42,990 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:43,107 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d3434ff01726422d95ac7db95a00defc_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d3434ff01726422d95ac7db95a00defc_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:43,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/4db917bf63ad490195ed1ccbf1b19cb6, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/4db917bf63ad490195ed1ccbf1b19cb6 is 175, key is test_row_0/A:col10/1732124921252/Put/seqid=0 2024-11-20T17:48:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742024_1200 (size=31255) 2024-11-20T17:48:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:43,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124983428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124983427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124983430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124983431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124983442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,445 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee28898c42e6b66b22f1f209861bee42 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124983537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124983537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124983537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,539 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/4db917bf63ad490195ed1ccbf1b19cb6 2024-11-20T17:48:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/c4117c3686794cb1a119664b95f4ad65 is 50, key is test_row_0/B:col10/1732124921252/Put/seqid=0 2024-11-20T17:48:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124983547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742025_1201 (size=12301) 2024-11-20T17:48:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124983747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124983757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:43,959 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/c4117c3686794cb1a119664b95f4ad65 2024-11-20T17:48:43,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/ae3aa949de2941eb85225fa3ef63af1b is 50, key is test_row_0/C:col10/1732124921252/Put/seqid=0 2024-11-20T17:48:43,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742026_1202 (size=12301) 2024-11-20T17:48:43,978 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/ae3aa949de2941eb85225fa3ef63af1b 2024-11-20T17:48:43,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/4db917bf63ad490195ed1ccbf1b19cb6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6 2024-11-20T17:48:43,991 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6, entries=150, sequenceid=276, filesize=30.5 K 2024-11-20T17:48:43,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/c4117c3686794cb1a119664b95f4ad65 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c4117c3686794cb1a119664b95f4ad65 2024-11-20T17:48:43,998 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c4117c3686794cb1a119664b95f4ad65, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T17:48:43,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/ae3aa949de2941eb85225fa3ef63af1b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/ae3aa949de2941eb85225fa3ef63af1b 2024-11-20T17:48:44,004 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/ae3aa949de2941eb85225fa3ef63af1b, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T17:48:44,005 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ee28898c42e6b66b22f1f209861bee42 in 1334ms, sequenceid=276, compaction requested=false 2024-11-20T17:48:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-20T17:48:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-20T17:48:44,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T17:48:44,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5680 sec 2024-11-20T17:48:44,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5730 sec 2024-11-20T17:48:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:44,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:44,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T17:48:44,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206652d911dcb44abb8518ca967dcf0d35_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:44,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742027_1203 (size=12454) 2024-11-20T17:48:44,078 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:44,087 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206652d911dcb44abb8518ca967dcf0d35_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206652d911dcb44abb8518ca967dcf0d35_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:44,088 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/1c82da1216b644d5b5075a17261a1a03, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:44,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/1c82da1216b644d5b5075a17261a1a03 is 175, key is test_row_0/A:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124984085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124984087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124984088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124984089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742028_1204 (size=31255) 2024-11-20T17:48:44,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/1c82da1216b644d5b5075a17261a1a03 2024-11-20T17:48:44,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/94112d43d5bf42adafe5a538321d5572 is 50, key is test_row_0/B:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:44,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742029_1205 (size=12301) 2024-11-20T17:48:44,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124984197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124984207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124984207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124984208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124984407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124984417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124984417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124984418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/94112d43d5bf42adafe5a538321d5572 2024-11-20T17:48:44,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/345b0ae50e3a4cf6956538c5df3246b5 is 50, key is test_row_0/C:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742030_1206 (size=12301) 2024-11-20T17:48:44,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124984717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124984727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124984727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124984727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:44,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:44,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/345b0ae50e3a4cf6956538c5df3246b5 2024-11-20T17:48:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/1c82da1216b644d5b5075a17261a1a03 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03 2024-11-20T17:48:44,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03, entries=150, sequenceid=294, filesize=30.5 K 2024-11-20T17:48:44,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/94112d43d5bf42adafe5a538321d5572 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/94112d43d5bf42adafe5a538321d5572 2024-11-20T17:48:44,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/94112d43d5bf42adafe5a538321d5572, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:48:44,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/345b0ae50e3a4cf6956538c5df3246b5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/345b0ae50e3a4cf6956538c5df3246b5 2024-11-20T17:48:44,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/345b0ae50e3a4cf6956538c5df3246b5, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:48:44,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ee28898c42e6b66b22f1f209861bee42 in 896ms, sequenceid=294, compaction requested=true 2024-11-20T17:48:44,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:44,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:48:44,957 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:44,958 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:44,958 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:44,958 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e7a176a1f08a4ed58bd463c6aca38993, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=92.0 K 2024-11-20T17:48:44,958 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:44,958 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/10249778768740dea4797ef0a1e7197f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c4117c3686794cb1a119664b95f4ad65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/94112d43d5bf42adafe5a538321d5572] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.5 K 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e7a176a1f08a4ed58bd463c6aca38993, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03] 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a176a1f08a4ed58bd463c6aca38993, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124920075 2024-11-20T17:48:44,958 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10249778768740dea4797ef0a1e7197f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124920075 2024-11-20T17:48:44,959 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4db917bf63ad490195ed1ccbf1b19cb6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732124921252 2024-11-20T17:48:44,959 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4117c3686794cb1a119664b95f4ad65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732124921252 2024-11-20T17:48:44,959 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c82da1216b644d5b5075a17261a1a03, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124924059 2024-11-20T17:48:44,959 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94112d43d5bf42adafe5a538321d5572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124924059 2024-11-20T17:48:44,969 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:44,970 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:44,971 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/32ad281d35ca487ba2ba361d86e81ccf is 50, key is test_row_0/B:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:44,971 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ad57e3bc62be40bf893dc0105a9af4b9_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:44,973 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ad57e3bc62be40bf893dc0105a9af4b9_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:44,973 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ad57e3bc62be40bf893dc0105a9af4b9_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:44,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742031_1207 (size=13017) 2024-11-20T17:48:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742032_1208 (size=4469) 2024-11-20T17:48:45,002 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#171 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:45,002 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/bcc599b4ee724d308445ffd7f453b3f5 is 175, key is test_row_0/A:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:45,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742033_1209 (size=31971) 2024-11-20T17:48:45,033 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/bcc599b4ee724d308445ffd7f453b3f5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bcc599b4ee724d308445ffd7f453b3f5 2024-11-20T17:48:45,042 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into bcc599b4ee724d308445ffd7f453b3f5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:45,042 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:45,042 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124924957; duration=0sec 2024-11-20T17:48:45,043 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:45,043 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:45,043 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:45,044 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:45,044 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:45,044 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:45,044 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/17b950182bca4275b63ab524ba5fdb60, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/ae3aa949de2941eb85225fa3ef63af1b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/345b0ae50e3a4cf6956538c5df3246b5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.5 K 2024-11-20T17:48:45,045 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 17b950182bca4275b63ab524ba5fdb60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124920075 2024-11-20T17:48:45,046 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3aa949de2941eb85225fa3ef63af1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732124921252 2024-11-20T17:48:45,047 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 345b0ae50e3a4cf6956538c5df3246b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124924059 2024-11-20T17:48:45,058 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:45,059 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/d7773ef8a55d48c685e94bf0e4e8186c is 50, key is test_row_0/C:col10/1732124924059/Put/seqid=0 2024-11-20T17:48:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742034_1210 (size=13017) 2024-11-20T17:48:45,078 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/d7773ef8a55d48c685e94bf0e4e8186c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d7773ef8a55d48c685e94bf0e4e8186c 2024-11-20T17:48:45,083 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into d7773ef8a55d48c685e94bf0e4e8186c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:45,083 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:45,083 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124924957; duration=0sec 2024-11-20T17:48:45,084 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:45,084 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:45,230 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:45,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T17:48:45,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204c5fa86db6ec4e429bdb0bd31f157cbc_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124924087/Put/seqid=0 2024-11-20T17:48:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124985248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124985250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124985251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124985253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742035_1211 (size=14994) 2024-11-20T17:48:45,266 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:45,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204c5fa86db6ec4e429bdb0bd31f157cbc_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204c5fa86db6ec4e429bdb0bd31f157cbc_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:45,275 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/52ccbfd114964f2995190cece6457a29, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:45,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/52ccbfd114964f2995190cece6457a29 is 175, key is test_row_0/A:col10/1732124924087/Put/seqid=0 2024-11-20T17:48:45,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742036_1212 (size=39949) 2024-11-20T17:48:45,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/52ccbfd114964f2995190cece6457a29 2024-11-20T17:48:45,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/e7ff3cb535f94664a9b1ba46ba192191 is 50, key is test_row_0/B:col10/1732124924087/Put/seqid=0 2024-11-20T17:48:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742037_1213 (size=12301) 2024-11-20T17:48:45,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/e7ff3cb535f94664a9b1ba46ba192191 2024-11-20T17:48:45,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/2b54decf0af24ce58f3447f74eead58a is 50, key is test_row_0/C:col10/1732124924087/Put/seqid=0 2024-11-20T17:48:45,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124985357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124985368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124985368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124985368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742038_1214 (size=12301) 2024-11-20T17:48:45,405 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/32ad281d35ca487ba2ba361d86e81ccf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/32ad281d35ca487ba2ba361d86e81ccf 2024-11-20T17:48:45,413 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into 32ad281d35ca487ba2ba361d86e81ccf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:45,413 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:45,413 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124924957; duration=0sec 2024-11-20T17:48:45,413 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:45,413 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:45,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124985467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,482 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee28898c42e6b66b22f1f209861bee42 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124985567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124985578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T17:48:45,618 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:45,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-20T17:48:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:45,621 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:45,622 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:45,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:45,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:45,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/2b54decf0af24ce58f3447f74eead58a 2024-11-20T17:48:45,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:45,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:45,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:45,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:45,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:45,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/52ccbfd114964f2995190cece6457a29 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29 2024-11-20T17:48:45,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29, entries=200, sequenceid=317, filesize=39.0 K 2024-11-20T17:48:45,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/e7ff3cb535f94664a9b1ba46ba192191 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/e7ff3cb535f94664a9b1ba46ba192191 2024-11-20T17:48:45,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/e7ff3cb535f94664a9b1ba46ba192191, entries=150, sequenceid=317, filesize=12.0 K 2024-11-20T17:48:45,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/2b54decf0af24ce58f3447f74eead58a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2b54decf0af24ce58f3447f74eead58a 2024-11-20T17:48:45,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2b54decf0af24ce58f3447f74eead58a, entries=150, sequenceid=317, filesize=12.0 K 2024-11-20T17:48:45,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for ee28898c42e6b66b22f1f209861bee42 in 570ms, sequenceid=317, compaction requested=false 2024-11-20T17:48:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:45,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T17:48:45,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205869ec299ff94183a81c2cfd408e0e7c_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:45,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742039_1215 (size=14994) 2024-11-20T17:48:45,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124985912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124985912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124985913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124985915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:45,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:45,919 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205869ec299ff94183a81c2cfd408e0e7c_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205869ec299ff94183a81c2cfd408e0e7c_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:45,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/21091f08bbb74335bb08f002c60101a4, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:45,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/21091f08bbb74335bb08f002c60101a4 is 175, key is test_row_0/A:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:45,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742040_1216 (size=39949) 2024-11-20T17:48:45,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/21091f08bbb74335bb08f002c60101a4 2024-11-20T17:48:45,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:45,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:45,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:45,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/f7b41c012c2a4447a71b438fc11a94f8 is 50, key is test_row_0/B:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:45,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742041_1217 (size=12301) 2024-11-20T17:48:46,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124986027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124986027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124986027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124986027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124986237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124986237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124986237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124986238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:46,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/f7b41c012c2a4447a71b438fc11a94f8 2024-11-20T17:48:46,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/31b4d4cd5ee84f9c888f3ae83ca37d49 is 50, key is test_row_0/C:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742042_1218 (size=12301) 2024-11-20T17:48:46,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124986547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124986547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124986547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124986548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:46,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:46,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/31b4d4cd5ee84f9c888f3ae83ca37d49 2024-11-20T17:48:46,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/21091f08bbb74335bb08f002c60101a4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4 2024-11-20T17:48:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:46,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4, entries=200, sequenceid=334, filesize=39.0 K 2024-11-20T17:48:46,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/f7b41c012c2a4447a71b438fc11a94f8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/f7b41c012c2a4447a71b438fc11a94f8 2024-11-20T17:48:46,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/f7b41c012c2a4447a71b438fc11a94f8, entries=150, sequenceid=334, filesize=12.0 K 2024-11-20T17:48:46,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/31b4d4cd5ee84f9c888f3ae83ca37d49 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/31b4d4cd5ee84f9c888f3ae83ca37d49 2024-11-20T17:48:46,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/31b4d4cd5ee84f9c888f3ae83ca37d49, entries=150, sequenceid=334, filesize=12.0 K 2024-11-20T17:48:46,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ee28898c42e6b66b22f1f209861bee42 in 884ms, sequenceid=334, compaction requested=true 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:46,775 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:46,775 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:46,776 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:46,776 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:46,776 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,776 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/32ad281d35ca487ba2ba361d86e81ccf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/e7ff3cb535f94664a9b1ba46ba192191, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/f7b41c012c2a4447a71b438fc11a94f8] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.7 K 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 32ad281d35ca487ba2ba361d86e81ccf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124924059 2024-11-20T17:48:46,777 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,777 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bcc599b4ee724d308445ffd7f453b3f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=109.2 K 2024-11-20T17:48:46,777 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ff3cb535f94664a9b1ba46ba192191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732124924076 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bcc599b4ee724d308445ffd7f453b3f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4] 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b41c012c2a4447a71b438fc11a94f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732124925243 2024-11-20T17:48:46,777 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc599b4ee724d308445ffd7f453b3f5, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124924059 2024-11-20T17:48:46,778 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ccbfd114964f2995190cece6457a29, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732124924076 2024-11-20T17:48:46,778 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21091f08bbb74335bb08f002c60101a4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732124925243 2024-11-20T17:48:46,786 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:46,788 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:46,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/12808f70a7164ecc9e33dd4f04ce4cd2 is 50, key is test_row_0/B:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:46,804 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120bea616214cb445c98ce10c50f83026a3_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:46,806 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120bea616214cb445c98ce10c50f83026a3_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:46,806 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bea616214cb445c98ce10c50f83026a3_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:46,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742043_1219 (size=13119) 2024-11-20T17:48:46,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742044_1220 (size=4469) 2024-11-20T17:48:46,833 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#180 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:46,834 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/cc5545a3cd54465f8ee0fa8abcac40cf is 175, key is test_row_0/A:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:46,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T17:48:46,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,847 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T17:48:46,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742045_1221 (size=32073) 2024-11-20T17:48:46,868 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/cc5545a3cd54465f8ee0fa8abcac40cf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/cc5545a3cd54465f8ee0fa8abcac40cf 2024-11-20T17:48:46,874 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into cc5545a3cd54465f8ee0fa8abcac40cf(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:46,874 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:46,874 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124926775; duration=0sec 2024-11-20T17:48:46,874 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:46,874 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:46,874 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:46,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120074a8444d00e4ef7a6886aed6105971b_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124925902/Put/seqid=0 2024-11-20T17:48:46,876 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:46,876 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:46,876 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:46,876 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d7773ef8a55d48c685e94bf0e4e8186c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2b54decf0af24ce58f3447f74eead58a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/31b4d4cd5ee84f9c888f3ae83ca37d49] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.7 K 2024-11-20T17:48:46,876 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7773ef8a55d48c685e94bf0e4e8186c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124924059 2024-11-20T17:48:46,877 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b54decf0af24ce58f3447f74eead58a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732124924076 2024-11-20T17:48:46,878 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31b4d4cd5ee84f9c888f3ae83ca37d49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732124925243 2024-11-20T17:48:46,895 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:46,895 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/847008e0a98846f6a6e4dc11a0260086 is 50, key is test_row_0/C:col10/1732124925243/Put/seqid=0 2024-11-20T17:48:46,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742046_1222 (size=12454) 2024-11-20T17:48:46,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:46,902 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120074a8444d00e4ef7a6886aed6105971b_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120074a8444d00e4ef7a6886aed6105971b_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/033a33c9af14457083f8f63a7d7a3ab8, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:46,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/033a33c9af14457083f8f63a7d7a3ab8 is 175, key is test_row_0/A:col10/1732124925902/Put/seqid=0 2024-11-20T17:48:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742047_1223 (size=13119) 2024-11-20T17:48:46,932 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/847008e0a98846f6a6e4dc11a0260086 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/847008e0a98846f6a6e4dc11a0260086 2024-11-20T17:48:46,939 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 847008e0a98846f6a6e4dc11a0260086(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:46,939 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:46,939 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124926775; duration=0sec 2024-11-20T17:48:46,939 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:46,939 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:46,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742048_1224 (size=31255) 2024-11-20T17:48:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124987069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124987070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124987070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124987071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124987177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124987177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124987187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124987187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,218 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/12808f70a7164ecc9e33dd4f04ce4cd2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/12808f70a7164ecc9e33dd4f04ce4cd2 2024-11-20T17:48:47,225 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into 12808f70a7164ecc9e33dd4f04ce4cd2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:47,225 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:47,225 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124926775; duration=0sec 2024-11-20T17:48:47,225 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:47,225 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:47,343 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/033a33c9af14457083f8f63a7d7a3ab8 2024-11-20T17:48:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/38068c9167ac481e9c786f687e431927 is 50, key is test_row_0/B:col10/1732124925902/Put/seqid=0 2024-11-20T17:48:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124987387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124987387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124987397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124987397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742049_1225 (size=12301) 2024-11-20T17:48:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124987697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124987697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124987707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124987707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:47,809 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/38068c9167ac481e9c786f687e431927 2024-11-20T17:48:47,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/783897eb771d446788a286b29f3cd1e6 is 50, key is test_row_0/C:col10/1732124925902/Put/seqid=0 2024-11-20T17:48:47,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742050_1226 (size=12301) 2024-11-20T17:48:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124988207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124988207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124988217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124988217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:48,221 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/783897eb771d446788a286b29f3cd1e6 2024-11-20T17:48:48,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/033a33c9af14457083f8f63a7d7a3ab8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8 2024-11-20T17:48:48,230 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8, entries=150, sequenceid=354, filesize=30.5 K 2024-11-20T17:48:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/38068c9167ac481e9c786f687e431927 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/38068c9167ac481e9c786f687e431927 2024-11-20T17:48:48,235 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/38068c9167ac481e9c786f687e431927, entries=150, sequenceid=354, filesize=12.0 K 2024-11-20T17:48:48,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/783897eb771d446788a286b29f3cd1e6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/783897eb771d446788a286b29f3cd1e6 2024-11-20T17:48:48,240 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/783897eb771d446788a286b29f3cd1e6, entries=150, sequenceid=354, filesize=12.0 K 2024-11-20T17:48:48,241 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ee28898c42e6b66b22f1f209861bee42 in 1394ms, sequenceid=354, compaction requested=false 2024-11-20T17:48:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-20T17:48:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-20T17:48:48,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T17:48:48,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6200 sec 2024-11-20T17:48:48,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.6250 sec 2024-11-20T17:48:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:49,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T17:48:49,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d6b0609f1af74621a5148809bf05dd6f_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:49,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742051_1227 (size=17534) 2024-11-20T17:48:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124989255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124989256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124989258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124989259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124989367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124989367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124989367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124989367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57652 deadline: 1732124989497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,502 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee28898c42e6b66b22f1f209861bee42 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:48:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124989577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124989577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124989577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124989577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,639 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:49,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d6b0609f1af74621a5148809bf05dd6f_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d6b0609f1af74621a5148809bf05dd6f_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:49,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/836022585c964e789cce486723d32a35, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:49,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/836022585c964e789cce486723d32a35 is 175, key is test_row_0/A:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:49,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742052_1228 (size=48639) 2024-11-20T17:48:49,657 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/836022585c964e789cce486723d32a35 2024-11-20T17:48:49,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/9793daaca1184cdd8994789ec2b224a3 is 50, key is test_row_0/B:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:49,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742053_1229 (size=12301) 2024-11-20T17:48:49,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/9793daaca1184cdd8994789ec2b224a3 2024-11-20T17:48:49,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/8a7ea838cd5242f6a9eb662054d37fe0 is 50, key is test_row_0/C:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:49,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742054_1230 (size=12301) 2024-11-20T17:48:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T17:48:49,787 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:49,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:48:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-20T17:48:49,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:48:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T17:48:49,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:48:49,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:48:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124989887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124989887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124989887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124989887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T17:48:49,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T17:48:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:49,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:50,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/8a7ea838cd5242f6a9eb662054d37fe0 2024-11-20T17:48:50,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/836022585c964e789cce486723d32a35 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35 2024-11-20T17:48:50,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35, entries=250, sequenceid=375, filesize=47.5 K 2024-11-20T17:48:50,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/9793daaca1184cdd8994789ec2b224a3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/9793daaca1184cdd8994789ec2b224a3 2024-11-20T17:48:50,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/9793daaca1184cdd8994789ec2b224a3, entries=150, sequenceid=375, filesize=12.0 K 2024-11-20T17:48:50,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T17:48:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:50,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:50,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:50,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/8a7ea838cd5242f6a9eb662054d37fe0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/8a7ea838cd5242f6a9eb662054d37fe0 2024-11-20T17:48:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:48:50,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/8a7ea838cd5242f6a9eb662054d37fe0, entries=150, sequenceid=375, filesize=12.0 K 2024-11-20T17:48:50,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ee28898c42e6b66b22f1f209861bee42 in 882ms, sequenceid=375, compaction requested=true 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:50,102 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:50,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:50,102 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:50,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:50,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:50,103 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,103 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/cc5545a3cd54465f8ee0fa8abcac40cf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=109.3 K 2024-11-20T17:48:50,103 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/cc5545a3cd54465f8ee0fa8abcac40cf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35] 2024-11-20T17:48:50,104 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:50,104 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5545a3cd54465f8ee0fa8abcac40cf, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732124925243 2024-11-20T17:48:50,104 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:50,104 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,104 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/12808f70a7164ecc9e33dd4f04ce4cd2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/38068c9167ac481e9c786f687e431927, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/9793daaca1184cdd8994789ec2b224a3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.8 K 2024-11-20T17:48:50,104 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 033a33c9af14457083f8f63a7d7a3ab8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732124925902 2024-11-20T17:48:50,105 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 12808f70a7164ecc9e33dd4f04ce4cd2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732124925243 2024-11-20T17:48:50,105 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 836022585c964e789cce486723d32a35, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732124927068 2024-11-20T17:48:50,105 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 38068c9167ac481e9c786f687e431927, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732124925902 2024-11-20T17:48:50,106 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9793daaca1184cdd8994789ec2b224a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732124927070 2024-11-20T17:48:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T17:48:50,114 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:50,115 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/942f4b2dbc62459faabaf38e352af373 is 50, key is test_row_0/B:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:50,121 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:50,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742055_1231 (size=13221) 2024-11-20T17:48:50,125 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112009afc61e4bbd4879a41d19a916ae45e1_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:50,128 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112009afc61e4bbd4879a41d19a916ae45e1_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:50,128 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112009afc61e4bbd4879a41d19a916ae45e1_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:50,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742056_1232 (size=4469) 2024-11-20T17:48:50,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T17:48:50,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,248 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T17:48:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203a6f1bbef0674e79ba5fc9d52cf24150_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124929226/Put/seqid=0 2024-11-20T17:48:50,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742057_1233 (size=12454) 2024-11-20T17:48:50,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. as already flushing 2024-11-20T17:48:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:50,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124990413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124990413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T17:48:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124990415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124990416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,528 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/942f4b2dbc62459faabaf38e352af373 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/942f4b2dbc62459faabaf38e352af373 2024-11-20T17:48:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124990527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124990527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124990527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124990527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,534 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into 942f4b2dbc62459faabaf38e352af373(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:50,534 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:50,534 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124930102; duration=0sec 2024-11-20T17:48:50,534 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:50,534 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:50,534 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:50,535 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:50,535 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:50,536 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,536 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/847008e0a98846f6a6e4dc11a0260086, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/783897eb771d446788a286b29f3cd1e6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/8a7ea838cd5242f6a9eb662054d37fe0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.8 K 2024-11-20T17:48:50,536 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 847008e0a98846f6a6e4dc11a0260086, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732124925243 2024-11-20T17:48:50,536 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 783897eb771d446788a286b29f3cd1e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732124925902 2024-11-20T17:48:50,537 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7ea838cd5242f6a9eb662054d37fe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732124927070 2024-11-20T17:48:50,542 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:50,543 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5cd4908011164650a855fe657f186119 is 50, key is test_row_0/C:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:50,545 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#189 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:50,546 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/ab2220bd5a794922941f03fcce4907b3 is 175, key is test_row_0/A:col10/1732124927070/Put/seqid=0 2024-11-20T17:48:50,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742058_1234 (size=13221) 2024-11-20T17:48:50,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742059_1235 (size=32175) 2024-11-20T17:48:50,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:50,663 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203a6f1bbef0674e79ba5fc9d52cf24150_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203a6f1bbef0674e79ba5fc9d52cf24150_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:50,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/82881de01a424872a2be8ce5999df934, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:50,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/82881de01a424872a2be8ce5999df934 is 175, key is test_row_0/A:col10/1732124929226/Put/seqid=0 2024-11-20T17:48:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742060_1236 (size=31255) 2024-11-20T17:48:50,669 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/82881de01a424872a2be8ce5999df934 2024-11-20T17:48:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/20e78e6b265040b99996ba41c01e904b is 50, key is test_row_0/B:col10/1732124929226/Put/seqid=0 2024-11-20T17:48:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742061_1237 (size=12301) 2024-11-20T17:48:50,691 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/20e78e6b265040b99996ba41c01e904b 2024-11-20T17:48:50,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/a0b3d1e0c1a14f1881d2ec2cc3942654 is 50, key is test_row_0/C:col10/1732124929226/Put/seqid=0 2024-11-20T17:48:50,719 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-20T17:48:50,719 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:50,719 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,720 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:50,721 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-20T17:48:50,721 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:50,721 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,721 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:50,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742062_1238 (size=12301) 2024-11-20T17:48:50,722 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-20T17:48:50,722 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:50,722 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,722 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-20T17:48:50,722 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:50,722 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,722 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:50,722 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:50,723 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:50,724 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/a0b3d1e0c1a14f1881d2ec2cc3942654 2024-11-20T17:48:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/82881de01a424872a2be8ce5999df934 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934 2024-11-20T17:48:50,733 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934, entries=150, sequenceid=393, filesize=30.5 K 2024-11-20T17:48:50,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/20e78e6b265040b99996ba41c01e904b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/20e78e6b265040b99996ba41c01e904b 2024-11-20T17:48:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57608 deadline: 1732124990737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1732124990737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1732124990737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:48:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57596 deadline: 1732124990737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,738 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/20e78e6b265040b99996ba41c01e904b, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:48:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:48:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/a0b3d1e0c1a14f1881d2ec2cc3942654 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/a0b3d1e0c1a14f1881d2ec2cc3942654 2024-11-20T17:48:50,743 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/a0b3d1e0c1a14f1881d2ec2cc3942654, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T17:48:50,743 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for ee28898c42e6b66b22f1f209861bee42 in 495ms, sequenceid=393, compaction requested=false 2024-11-20T17:48:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:50,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-20T17:48:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-20T17:48:50,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T17:48:50,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 955 msec 2024-11-20T17:48:50,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 959 msec 2024-11-20T17:48:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T17:48:50,927 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:48:50,953 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5cd4908011164650a855fe657f186119 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5cd4908011164650a855fe657f186119 2024-11-20T17:48:50,957 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/ab2220bd5a794922941f03fcce4907b3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ab2220bd5a794922941f03fcce4907b3 2024-11-20T17:48:50,957 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 5cd4908011164650a855fe657f186119(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:50,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:50,957 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124930102; duration=0sec 2024-11-20T17:48:50,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:50,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:50,962 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into ab2220bd5a794922941f03fcce4907b3(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:50,962 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:50,962 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124930102; duration=0sec 2024-11-20T17:48:50,962 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:50,962 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:51,049 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:48:51,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T17:48:51,049 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-20T17:48:51,049 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:51,049 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-20T17:48:51,049 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,049 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-20T17:48:51,049 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:51,049 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:51,049 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,049 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,049 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:51,049 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,049 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:51,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:51,051 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-20T17:48:51,051 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:51,051 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,051 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:51,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:51,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f1ec13efaf8244b5a5e2889383456553_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_0/A:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:51,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742063_1239 (size=12454) 2024-11-20T17:48:51,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:48:51,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:48:51,465 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f1ec13efaf8244b5a5e2889383456553_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f1ec13efaf8244b5a5e2889383456553_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:51,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:51,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2 is 175, key is test_row_0/A:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:51,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742064_1240 (size=31255) 2024-11-20T17:48:51,871 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2 2024-11-20T17:48:51,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/aa447d6121c64d55a851552168c4bfaf is 50, key is test_row_0/B:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:51,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742065_1241 (size=12301) 2024-11-20T17:48:52,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/aa447d6121c64d55a851552168c4bfaf 2024-11-20T17:48:52,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/e212e88b38ff4adf915d180f7a9db7b5 is 50, key is test_row_0/C:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:52,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742066_1242 (size=12301) 2024-11-20T17:48:52,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/e212e88b38ff4adf915d180f7a9db7b5 2024-11-20T17:48:52,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2 2024-11-20T17:48:52,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2, entries=150, sequenceid=418, filesize=30.5 K 2024-11-20T17:48:52,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/aa447d6121c64d55a851552168c4bfaf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa447d6121c64d55a851552168c4bfaf 2024-11-20T17:48:52,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa447d6121c64d55a851552168c4bfaf, entries=150, sequenceid=418, filesize=12.0 K 2024-11-20T17:48:52,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/e212e88b38ff4adf915d180f7a9db7b5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/e212e88b38ff4adf915d180f7a9db7b5 2024-11-20T17:48:52,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/e212e88b38ff4adf915d180f7a9db7b5, entries=150, sequenceid=418, filesize=12.0 K 2024-11-20T17:48:52,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for ee28898c42e6b66b22f1f209861bee42 in 1675ms, sequenceid=418, compaction requested=true 2024-11-20T17:48:52,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:52,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:48:52,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:52,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:48:52,725 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:52,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:52,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee28898c42e6b66b22f1f209861bee42:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:48:52,725 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:52,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:52,726 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:52,726 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:52,726 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/A is initiating minor compaction (all files) 2024-11-20T17:48:52,726 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/B is initiating minor compaction (all files) 2024-11-20T17:48:52,726 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/A in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:52,726 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/B in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:52,726 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ab2220bd5a794922941f03fcce4907b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=92.5 K 2024-11-20T17:48:52,726 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/942f4b2dbc62459faabaf38e352af373, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/20e78e6b265040b99996ba41c01e904b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa447d6121c64d55a851552168c4bfaf] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.9 K 2024-11-20T17:48:52,726 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:52,726 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ab2220bd5a794922941f03fcce4907b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2] 2024-11-20T17:48:52,727 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 942f4b2dbc62459faabaf38e352af373, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732124927070 2024-11-20T17:48:52,727 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab2220bd5a794922941f03fcce4907b3, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732124927070 2024-11-20T17:48:52,727 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e78e6b265040b99996ba41c01e904b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732124929226 2024-11-20T17:48:52,727 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82881de01a424872a2be8ce5999df934, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732124929226 2024-11-20T17:48:52,727 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting aa447d6121c64d55a851552168c4bfaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732124930410 2024-11-20T17:48:52,728 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d05e3352ac5d4397b6a0e5dbbfcc5bb2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732124930410 2024-11-20T17:48:52,738 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#B#compaction#197 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:52,738 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/cade831583554f669f667fcc1ca1073f is 50, key is test_row_0/B:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:52,741 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:52,745 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a472489f1cc44008b861d2221180673b_ee28898c42e6b66b22f1f209861bee42 store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:52,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742067_1243 (size=13323) 2024-11-20T17:48:52,751 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/cade831583554f669f667fcc1ca1073f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/cade831583554f669f667fcc1ca1073f 2024-11-20T17:48:52,757 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/B of ee28898c42e6b66b22f1f209861bee42 into cade831583554f669f667fcc1ca1073f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:52,757 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:52,757 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/B, priority=13, startTime=1732124932724; duration=0sec 2024-11-20T17:48:52,758 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:48:52,758 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:B 2024-11-20T17:48:52,758 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:48:52,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:48:52,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee28898c42e6b66b22f1f209861bee42/C is initiating minor compaction (all files) 2024-11-20T17:48:52,759 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee28898c42e6b66b22f1f209861bee42/C in TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:52,759 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5cd4908011164650a855fe657f186119, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/a0b3d1e0c1a14f1881d2ec2cc3942654, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/e212e88b38ff4adf915d180f7a9db7b5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp, totalSize=36.9 K 2024-11-20T17:48:52,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd4908011164650a855fe657f186119, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732124927070 2024-11-20T17:48:52,760 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b3d1e0c1a14f1881d2ec2cc3942654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732124929226 2024-11-20T17:48:52,760 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e212e88b38ff4adf915d180f7a9db7b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732124930410 2024-11-20T17:48:52,775 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#C#compaction#199 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:52,775 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5aecf2f0926442a6b9964731c1f4d289 is 50, key is test_row_0/C:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:52,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742068_1244 (size=13323) 2024-11-20T17:48:52,781 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a472489f1cc44008b861d2221180673b_ee28898c42e6b66b22f1f209861bee42, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:52,781 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a472489f1cc44008b861d2221180673b_ee28898c42e6b66b22f1f209861bee42 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:48:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742069_1245 (size=4469) 2024-11-20T17:48:53,186 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/5aecf2f0926442a6b9964731c1f4d289 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5aecf2f0926442a6b9964731c1f4d289 2024-11-20T17:48:53,187 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee28898c42e6b66b22f1f209861bee42#A#compaction#198 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:48:53,188 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/f82a9dc207d740fa82c8b0dd169a3986 is 175, key is test_row_0/A:col10/1732124931049/Put/seqid=0 2024-11-20T17:48:53,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742070_1246 (size=32277) 2024-11-20T17:48:53,193 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/C of ee28898c42e6b66b22f1f209861bee42 into 5aecf2f0926442a6b9964731c1f4d289(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:53,193 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:53,193 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/C, priority=13, startTime=1732124932725; duration=0sec 2024-11-20T17:48:53,193 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:53,193 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:C 2024-11-20T17:48:53,599 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/f82a9dc207d740fa82c8b0dd169a3986 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/f82a9dc207d740fa82c8b0dd169a3986 2024-11-20T17:48:53,604 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee28898c42e6b66b22f1f209861bee42/A of ee28898c42e6b66b22f1f209861bee42 into f82a9dc207d740fa82c8b0dd169a3986(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:48:53,605 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee28898c42e6b66b22f1f209861bee42: 2024-11-20T17:48:53,605 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., storeName=ee28898c42e6b66b22f1f209861bee42/A, priority=13, startTime=1732124932724; duration=0sec 2024-11-20T17:48:53,605 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:48:53,605 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee28898c42e6b66b22f1f209861bee42:A 2024-11-20T17:48:59,542 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-20T17:48:59,542 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:48:59,542 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:59,542 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:59,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:59,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T17:48:59,543 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3634 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3565 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1569 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4707 rows 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1564 2024-11-20T17:48:59,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4692 rows 2024-11-20T17:48:59,544 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:48:59,545 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:48:59,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:59,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:48:59,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:48:59,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T17:48:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T17:48:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T17:48:59,550 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124939550"}]},"ts":"1732124939550"} 2024-11-20T17:48:59,552 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T17:48:59,552 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T17:48:59,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:48:59,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, UNASSIGN}] 2024-11-20T17:48:59,555 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, UNASSIGN 2024-11-20T17:48:59,556 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:48:59,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, UNASSIGN because future has completed 2024-11-20T17:48:59,557 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:48:59,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:48:59,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T17:48:59,710 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:48:59,711 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:48:59,711 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing ee28898c42e6b66b22f1f209861bee42, disabling compactions & flushes 2024-11-20T17:48:59,711 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:59,711 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:59,711 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. after waiting 0 ms 2024-11-20T17:48:59,712 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:48:59,712 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing ee28898c42e6b66b22f1f209861bee42 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-20T17:48:59,723 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c5574409f38b4cffa0d5b60eb6620a3e_ee28898c42e6b66b22f1f209861bee42 is 50, key is test_row_1/A:col10/1732124939539/Put/seqid=0 2024-11-20T17:48:59,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742071_1247 (size=7374) 2024-11-20T17:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T17:49:00,128 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:00,133 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c5574409f38b4cffa0d5b60eb6620a3e_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c5574409f38b4cffa0d5b60eb6620a3e_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:00,134 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/549c1f7d26894f40b1f2bd5317a7a84d, store: [table=TestAcidGuarantees family=A region=ee28898c42e6b66b22f1f209861bee42] 2024-11-20T17:49:00,135 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/549c1f7d26894f40b1f2bd5317a7a84d is 175, key is test_row_1/A:col10/1732124939539/Put/seqid=0 2024-11-20T17:49:00,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742072_1248 (size=13865) 2024-11-20T17:49:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T17:49:00,539 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/549c1f7d26894f40b1f2bd5317a7a84d 2024-11-20T17:49:00,547 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/5567c17cf4764747bdf3fa4488264f2b is 50, key is test_row_1/B:col10/1732124939539/Put/seqid=0 2024-11-20T17:49:00,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742073_1249 (size=7415) 2024-11-20T17:49:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T17:49:00,951 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/5567c17cf4764747bdf3fa4488264f2b 2024-11-20T17:49:00,960 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/de7239c2481c461da8a242369cd277cb is 50, key is test_row_1/C:col10/1732124939539/Put/seqid=0 2024-11-20T17:49:00,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742074_1250 (size=7415) 2024-11-20T17:49:01,364 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/de7239c2481c461da8a242369cd277cb 2024-11-20T17:49:01,368 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/A/549c1f7d26894f40b1f2bd5317a7a84d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/549c1f7d26894f40b1f2bd5317a7a84d 2024-11-20T17:49:01,373 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/549c1f7d26894f40b1f2bd5317a7a84d, entries=50, sequenceid=425, filesize=13.5 K 2024-11-20T17:49:01,374 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/B/5567c17cf4764747bdf3fa4488264f2b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/5567c17cf4764747bdf3fa4488264f2b 2024-11-20T17:49:01,378 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/5567c17cf4764747bdf3fa4488264f2b, entries=50, sequenceid=425, filesize=7.2 K 2024-11-20T17:49:01,379 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/.tmp/C/de7239c2481c461da8a242369cd277cb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/de7239c2481c461da8a242369cd277cb 2024-11-20T17:49:01,384 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/de7239c2481c461da8a242369cd277cb, entries=50, sequenceid=425, filesize=7.2 K 2024-11-20T17:49:01,385 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ee28898c42e6b66b22f1f209861bee42 in 1673ms, sequenceid=425, compaction requested=false 2024-11-20T17:49:01,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0dd7affbf2e54c358326f7c414d9aea9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0f2dab3fb7744880ac78586af1b58442, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0c09664bd6d4c7aa91a73569f1ad143, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0eddafe9c2b443087f0e89835cfe8c8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c8841e97b9e24273a11c926253cb3ea0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e7a176a1f08a4ed58bd463c6aca38993, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bcc599b4ee724d308445ffd7f453b3f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/cc5545a3cd54465f8ee0fa8abcac40cf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ab2220bd5a794922941f03fcce4907b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2] to archive 2024-11-20T17:49:01,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:01,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/580ddb2af54141fd924b30a6fd5d3cfb 2024-11-20T17:49:01,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/09753210871e424f9076d5e329a68276 2024-11-20T17:49:01,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/6845d4e978ba4e8686aceb1badee7d36 2024-11-20T17:49:01,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0dd7affbf2e54c358326f7c414d9aea9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0dd7affbf2e54c358326f7c414d9aea9 2024-11-20T17:49:01,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/7ab528b58aa448a4ab5e71e5ce97f6bd 2024-11-20T17:49:01,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/139210c22f3d4f15975d2dcd7591cb38 2024-11-20T17:49:01,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0f2dab3fb7744880ac78586af1b58442 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0f2dab3fb7744880ac78586af1b58442 2024-11-20T17:49:01,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bd115ce7099b430f9a8e925096ecbb21 2024-11-20T17:49:01,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0c09664bd6d4c7aa91a73569f1ad143 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0c09664bd6d4c7aa91a73569f1ad143 2024-11-20T17:49:01,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/0477a678cf64486d80ab2293b29b587c 2024-11-20T17:49:01,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ee8a223a733f4f619b27e92896c75016 2024-11-20T17:49:01,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/00742f98d7944e18b967e0bbc466c329 2024-11-20T17:49:01,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0eddafe9c2b443087f0e89835cfe8c8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c0eddafe9c2b443087f0e89835cfe8c8 2024-11-20T17:49:01,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/640cd2e9e28e4be498246481c6020ff6 2024-11-20T17:49:01,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/250d5d15cb814792b7773bdcd45617a0 2024-11-20T17:49:01,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c8841e97b9e24273a11c926253cb3ea0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/c8841e97b9e24273a11c926253cb3ea0 2024-11-20T17:49:01,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e42746dcd6a145b2b1fdc06719b30bbc 2024-11-20T17:49:01,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/324d50479dde4dea838fa63c73581be3 2024-11-20T17:49:01,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e7a176a1f08a4ed58bd463c6aca38993 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/e7a176a1f08a4ed58bd463c6aca38993 2024-11-20T17:49:01,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/4db917bf63ad490195ed1ccbf1b19cb6 2024-11-20T17:49:01,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bcc599b4ee724d308445ffd7f453b3f5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/bcc599b4ee724d308445ffd7f453b3f5 2024-11-20T17:49:01,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/1c82da1216b644d5b5075a17261a1a03 2024-11-20T17:49:01,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/52ccbfd114964f2995190cece6457a29 2024-11-20T17:49:01,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/21091f08bbb74335bb08f002c60101a4 2024-11-20T17:49:01,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/cc5545a3cd54465f8ee0fa8abcac40cf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/cc5545a3cd54465f8ee0fa8abcac40cf 2024-11-20T17:49:01,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/033a33c9af14457083f8f63a7d7a3ab8 2024-11-20T17:49:01,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/836022585c964e789cce486723d32a35 2024-11-20T17:49:01,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ab2220bd5a794922941f03fcce4907b3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/ab2220bd5a794922941f03fcce4907b3 2024-11-20T17:49:01,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/82881de01a424872a2be8ce5999df934 2024-11-20T17:49:01,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/d05e3352ac5d4397b6a0e5dbbfcc5bb2 2024-11-20T17:49:01,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c2688ca1cc9b40f59af7678d02db277d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/226298bff2c44383885ab5c532390f4b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/fb5788f9806b4e56b6b7c01860600ec1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc26d5ccb46446b89374b5d450e48865, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/6d10a257b9d74c8fb4aae198cb0b1675, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa90b045856e42c981cbb8a9f155f555, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8277e362faa34b7d906b46d3e6e1dfe9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/ece1063b31b741c4b386b9f7a336ee49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/2bed0d05e5cf4000bf10b7ad34301bf4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8d2d5c2efcb64d548460d0f59e2ce97a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0547fbf618f74915b6ebde8be9682288, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/b8a80dfb44b74a3197da810cd82e5fe8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/18d4f5f18ef5451d99ab9b248155208c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc06b2c9911c4c289b83bbbf90a40d7e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/711226785f3342519a38ab1cea3f1a34, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/071058472ec94349a938320b81cba457, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/302592a2ce84477194a8e613d3611ddd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/10249778768740dea4797ef0a1e7197f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0a1af0d3c8514aab88c45ae58c5893d5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c4117c3686794cb1a119664b95f4ad65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/32ad281d35ca487ba2ba361d86e81ccf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/94112d43d5bf42adafe5a538321d5572, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/e7ff3cb535f94664a9b1ba46ba192191, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/12808f70a7164ecc9e33dd4f04ce4cd2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/f7b41c012c2a4447a71b438fc11a94f8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/38068c9167ac481e9c786f687e431927, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/942f4b2dbc62459faabaf38e352af373, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/9793daaca1184cdd8994789ec2b224a3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/20e78e6b265040b99996ba41c01e904b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa447d6121c64d55a851552168c4bfaf] to archive 2024-11-20T17:49:01,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:01,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c2688ca1cc9b40f59af7678d02db277d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c2688ca1cc9b40f59af7678d02db277d 2024-11-20T17:49:01,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/226298bff2c44383885ab5c532390f4b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/226298bff2c44383885ab5c532390f4b 2024-11-20T17:49:01,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/fb5788f9806b4e56b6b7c01860600ec1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/fb5788f9806b4e56b6b7c01860600ec1 2024-11-20T17:49:01,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc26d5ccb46446b89374b5d450e48865 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc26d5ccb46446b89374b5d450e48865 2024-11-20T17:49:01,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/6d10a257b9d74c8fb4aae198cb0b1675 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/6d10a257b9d74c8fb4aae198cb0b1675 2024-11-20T17:49:01,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa90b045856e42c981cbb8a9f155f555 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa90b045856e42c981cbb8a9f155f555 2024-11-20T17:49:01,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8277e362faa34b7d906b46d3e6e1dfe9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8277e362faa34b7d906b46d3e6e1dfe9 2024-11-20T17:49:01,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/ece1063b31b741c4b386b9f7a336ee49 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/ece1063b31b741c4b386b9f7a336ee49 2024-11-20T17:49:01,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/2bed0d05e5cf4000bf10b7ad34301bf4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/2bed0d05e5cf4000bf10b7ad34301bf4 2024-11-20T17:49:01,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8d2d5c2efcb64d548460d0f59e2ce97a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/8d2d5c2efcb64d548460d0f59e2ce97a 2024-11-20T17:49:01,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0547fbf618f74915b6ebde8be9682288 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0547fbf618f74915b6ebde8be9682288 2024-11-20T17:49:01,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/b8a80dfb44b74a3197da810cd82e5fe8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/b8a80dfb44b74a3197da810cd82e5fe8 2024-11-20T17:49:01,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/18d4f5f18ef5451d99ab9b248155208c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/18d4f5f18ef5451d99ab9b248155208c 2024-11-20T17:49:01,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc06b2c9911c4c289b83bbbf90a40d7e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/bc06b2c9911c4c289b83bbbf90a40d7e 2024-11-20T17:49:01,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/711226785f3342519a38ab1cea3f1a34 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/711226785f3342519a38ab1cea3f1a34 2024-11-20T17:49:01,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/071058472ec94349a938320b81cba457 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/071058472ec94349a938320b81cba457 2024-11-20T17:49:01,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/302592a2ce84477194a8e613d3611ddd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/302592a2ce84477194a8e613d3611ddd 2024-11-20T17:49:01,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/10249778768740dea4797ef0a1e7197f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/10249778768740dea4797ef0a1e7197f 2024-11-20T17:49:01,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0a1af0d3c8514aab88c45ae58c5893d5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/0a1af0d3c8514aab88c45ae58c5893d5 2024-11-20T17:49:01,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c4117c3686794cb1a119664b95f4ad65 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/c4117c3686794cb1a119664b95f4ad65 2024-11-20T17:49:01,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/32ad281d35ca487ba2ba361d86e81ccf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/32ad281d35ca487ba2ba361d86e81ccf 2024-11-20T17:49:01,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/94112d43d5bf42adafe5a538321d5572 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/94112d43d5bf42adafe5a538321d5572 2024-11-20T17:49:01,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/e7ff3cb535f94664a9b1ba46ba192191 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/e7ff3cb535f94664a9b1ba46ba192191 2024-11-20T17:49:01,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/12808f70a7164ecc9e33dd4f04ce4cd2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/12808f70a7164ecc9e33dd4f04ce4cd2 2024-11-20T17:49:01,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/f7b41c012c2a4447a71b438fc11a94f8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/f7b41c012c2a4447a71b438fc11a94f8 2024-11-20T17:49:01,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/38068c9167ac481e9c786f687e431927 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/38068c9167ac481e9c786f687e431927 2024-11-20T17:49:01,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/942f4b2dbc62459faabaf38e352af373 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/942f4b2dbc62459faabaf38e352af373 2024-11-20T17:49:01,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/9793daaca1184cdd8994789ec2b224a3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/9793daaca1184cdd8994789ec2b224a3 2024-11-20T17:49:01,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/20e78e6b265040b99996ba41c01e904b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/20e78e6b265040b99996ba41c01e904b 2024-11-20T17:49:01,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa447d6121c64d55a851552168c4bfaf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/aa447d6121c64d55a851552168c4bfaf 2024-11-20T17:49:01,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cf2ea60f46344714b4a47126ddfdd942, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/82feefe28c4e4ae29c487dd84ad01c28, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/6cba74e13b3349adaef1882a144c37f2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/883050b3526b44a4b21be8e1adc1dafd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/33566c5e9e6f46989105b3308d8f8dd0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/4e544b8d93144013af0d4e31ec4e1ff1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d8d118d4b3fc4b488f404ec20fce4a46, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5a162ce09f2246c286a3a47940fd15d3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/35c6793e2c9b41a6bf0cc253ed86ac7e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/411511de4aac4fb199a0fe02758d1598, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/f9f6cd3d4bf04a28a01bde3cb1cb3714, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/98e71eb3b19f4ef49cc505c981297fbc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/01849e6141174b2ca0e9557e7546753c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/397234fd99334732aecff7c35a9acbca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/37652b68b44740078d87557ab0a08328, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2fe7ecc7a9524a00876faeb3fedd4dd2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/1dacf0c2926644bf90352a6acc3a1c7a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/17b950182bca4275b63ab524ba5fdb60, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cb8180ea0277465aa4c8bab34d05e568, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/ae3aa949de2941eb85225fa3ef63af1b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d7773ef8a55d48c685e94bf0e4e8186c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/345b0ae50e3a4cf6956538c5df3246b5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2b54decf0af24ce58f3447f74eead58a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/847008e0a98846f6a6e4dc11a0260086, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/31b4d4cd5ee84f9c888f3ae83ca37d49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/783897eb771d446788a286b29f3cd1e6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5cd4908011164650a855fe657f186119, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/8a7ea838cd5242f6a9eb662054d37fe0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/a0b3d1e0c1a14f1881d2ec2cc3942654, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/e212e88b38ff4adf915d180f7a9db7b5] to archive 2024-11-20T17:49:01,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:01,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cf2ea60f46344714b4a47126ddfdd942 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cf2ea60f46344714b4a47126ddfdd942 2024-11-20T17:49:01,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/82feefe28c4e4ae29c487dd84ad01c28 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/82feefe28c4e4ae29c487dd84ad01c28 2024-11-20T17:49:01,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/6cba74e13b3349adaef1882a144c37f2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/6cba74e13b3349adaef1882a144c37f2 2024-11-20T17:49:01,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/883050b3526b44a4b21be8e1adc1dafd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/883050b3526b44a4b21be8e1adc1dafd 2024-11-20T17:49:01,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/33566c5e9e6f46989105b3308d8f8dd0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/33566c5e9e6f46989105b3308d8f8dd0 2024-11-20T17:49:01,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/4e544b8d93144013af0d4e31ec4e1ff1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/4e544b8d93144013af0d4e31ec4e1ff1 2024-11-20T17:49:01,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d8d118d4b3fc4b488f404ec20fce4a46 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d8d118d4b3fc4b488f404ec20fce4a46 2024-11-20T17:49:01,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5a162ce09f2246c286a3a47940fd15d3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5a162ce09f2246c286a3a47940fd15d3 2024-11-20T17:49:01,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/35c6793e2c9b41a6bf0cc253ed86ac7e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/35c6793e2c9b41a6bf0cc253ed86ac7e 2024-11-20T17:49:01,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/411511de4aac4fb199a0fe02758d1598 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/411511de4aac4fb199a0fe02758d1598 2024-11-20T17:49:01,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/f9f6cd3d4bf04a28a01bde3cb1cb3714 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/f9f6cd3d4bf04a28a01bde3cb1cb3714 2024-11-20T17:49:01,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/98e71eb3b19f4ef49cc505c981297fbc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/98e71eb3b19f4ef49cc505c981297fbc 2024-11-20T17:49:01,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/01849e6141174b2ca0e9557e7546753c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/01849e6141174b2ca0e9557e7546753c 2024-11-20T17:49:01,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/397234fd99334732aecff7c35a9acbca to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/397234fd99334732aecff7c35a9acbca 2024-11-20T17:49:01,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/37652b68b44740078d87557ab0a08328 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/37652b68b44740078d87557ab0a08328 2024-11-20T17:49:01,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2fe7ecc7a9524a00876faeb3fedd4dd2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2fe7ecc7a9524a00876faeb3fedd4dd2 2024-11-20T17:49:01,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/1dacf0c2926644bf90352a6acc3a1c7a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/1dacf0c2926644bf90352a6acc3a1c7a 2024-11-20T17:49:01,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/17b950182bca4275b63ab524ba5fdb60 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/17b950182bca4275b63ab524ba5fdb60 2024-11-20T17:49:01,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cb8180ea0277465aa4c8bab34d05e568 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/cb8180ea0277465aa4c8bab34d05e568 2024-11-20T17:49:01,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/ae3aa949de2941eb85225fa3ef63af1b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/ae3aa949de2941eb85225fa3ef63af1b 2024-11-20T17:49:01,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d7773ef8a55d48c685e94bf0e4e8186c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/d7773ef8a55d48c685e94bf0e4e8186c 2024-11-20T17:49:01,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/345b0ae50e3a4cf6956538c5df3246b5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/345b0ae50e3a4cf6956538c5df3246b5 2024-11-20T17:49:01,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2b54decf0af24ce58f3447f74eead58a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/2b54decf0af24ce58f3447f74eead58a 2024-11-20T17:49:01,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/847008e0a98846f6a6e4dc11a0260086 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/847008e0a98846f6a6e4dc11a0260086 2024-11-20T17:49:01,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/31b4d4cd5ee84f9c888f3ae83ca37d49 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/31b4d4cd5ee84f9c888f3ae83ca37d49 2024-11-20T17:49:01,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/783897eb771d446788a286b29f3cd1e6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/783897eb771d446788a286b29f3cd1e6 2024-11-20T17:49:01,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5cd4908011164650a855fe657f186119 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5cd4908011164650a855fe657f186119 2024-11-20T17:49:01,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/8a7ea838cd5242f6a9eb662054d37fe0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/8a7ea838cd5242f6a9eb662054d37fe0 2024-11-20T17:49:01,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/a0b3d1e0c1a14f1881d2ec2cc3942654 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/a0b3d1e0c1a14f1881d2ec2cc3942654 2024-11-20T17:49:01,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/e212e88b38ff4adf915d180f7a9db7b5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/e212e88b38ff4adf915d180f7a9db7b5 2024-11-20T17:49:01,535 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/recovered.edits/428.seqid, newMaxSeqId=428, maxSeqId=4 2024-11-20T17:49:01,536 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. 2024-11-20T17:49:01,537 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for ee28898c42e6b66b22f1f209861bee42: Waiting for close lock at 1732124939711Running coprocessor pre-close hooks at 1732124939711Disabling compacts and flushes for region at 1732124939711Disabling writes for close at 1732124939712 (+1 ms)Obtaining lock to block concurrent updates at 1732124939712Preparing flush snapshotting stores in ee28898c42e6b66b22f1f209861bee42 at 1732124939712Finished memstore snapshotting TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732124939713 (+1 ms)Flushing stores of TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42. at 1732124939714 (+1 ms)Flushing ee28898c42e6b66b22f1f209861bee42/A: creating writer at 1732124939714Flushing mob file ee28898c42e6b66b22f1f209861bee42/A: appending metadata at 1732124939723 (+9 ms)Flushing mob file ee28898c42e6b66b22f1f209861bee42/A: closing flushed file at 1732124939723Flushing ee28898c42e6b66b22f1f209861bee42/A: appending metadata at 1732124940134 (+411 ms)Flushing ee28898c42e6b66b22f1f209861bee42/A: closing flushed file at 1732124940134Flushing ee28898c42e6b66b22f1f209861bee42/B: creating writer at 1732124940543 (+409 ms)Flushing ee28898c42e6b66b22f1f209861bee42/B: appending metadata at 1732124940546 (+3 ms)Flushing ee28898c42e6b66b22f1f209861bee42/B: closing flushed file at 1732124940546Flushing ee28898c42e6b66b22f1f209861bee42/C: creating writer at 1732124940956 (+410 ms)Flushing ee28898c42e6b66b22f1f209861bee42/C: appending metadata at 1732124940959 (+3 ms)Flushing ee28898c42e6b66b22f1f209861bee42/C: closing flushed file at 1732124940959Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48dadf94: reopening flushed file at 1732124941368 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4af84fdf: reopening flushed file at 1732124941373 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1977040a: reopening flushed file at 1732124941378 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ee28898c42e6b66b22f1f209861bee42 in 1673ms, sequenceid=425, compaction requested=false at 1732124941386 (+8 ms)Writing region close event to WAL at 1732124941524 (+138 ms)Running coprocessor post-close hooks at 1732124941536 (+12 ms)Closed at 1732124941536 2024-11-20T17:49:01,539 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,540 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=ee28898c42e6b66b22f1f209861bee42, regionState=CLOSED 2024-11-20T17:49:01,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:49:01,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T17:49:01,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure ee28898c42e6b66b22f1f209861bee42, server=613581fcdc40,44957,1732124872893 in 1.9920 sec 2024-11-20T17:49:01,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-20T17:49:01,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee28898c42e6b66b22f1f209861bee42, UNASSIGN in 1.9970 sec 2024-11-20T17:49:01,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T17:49:01,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0010 sec 2024-11-20T17:49:01,558 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124941558"}]},"ts":"1732124941558"} 2024-11-20T17:49:01,559 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T17:49:01,559 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T17:49:01,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0130 sec 2024-11-20T17:49:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T17:49:01,698 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:01,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T17:49:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,700 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T17:49:01,701 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,705 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,707 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/recovered.edits] 2024-11-20T17:49:01,711 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/549c1f7d26894f40b1f2bd5317a7a84d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/549c1f7d26894f40b1f2bd5317a7a84d 2024-11-20T17:49:01,712 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/f82a9dc207d740fa82c8b0dd169a3986 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/A/f82a9dc207d740fa82c8b0dd169a3986 2024-11-20T17:49:01,715 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/5567c17cf4764747bdf3fa4488264f2b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/5567c17cf4764747bdf3fa4488264f2b 2024-11-20T17:49:01,716 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/cade831583554f669f667fcc1ca1073f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/B/cade831583554f669f667fcc1ca1073f 2024-11-20T17:49:01,718 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5aecf2f0926442a6b9964731c1f4d289 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/5aecf2f0926442a6b9964731c1f4d289 2024-11-20T17:49:01,719 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/de7239c2481c461da8a242369cd277cb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/C/de7239c2481c461da8a242369cd277cb 2024-11-20T17:49:01,721 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/recovered.edits/428.seqid to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42/recovered.edits/428.seqid 2024-11-20T17:49:01,722 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,722 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T17:49:01,722 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T17:49:01,723 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T17:49:01,733 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120074a8444d00e4ef7a6886aed6105971b_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120074a8444d00e4ef7a6886aed6105971b_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,738 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ddd179e78a94ff18ee59d23dabcd8ae_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ddd179e78a94ff18ee59d23dabcd8ae_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,741 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203a6f1bbef0674e79ba5fc9d52cf24150_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203a6f1bbef0674e79ba5fc9d52cf24150_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,743 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204c5fa86db6ec4e429bdb0bd31f157cbc_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204c5fa86db6ec4e429bdb0bd31f157cbc_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,745 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112057b1b72fad6f4e4b9e0ac2dfa4501897_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112057b1b72fad6f4e4b9e0ac2dfa4501897_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,751 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205869ec299ff94183a81c2cfd408e0e7c_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205869ec299ff94183a81c2cfd408e0e7c_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,752 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206652d911dcb44abb8518ca967dcf0d35_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206652d911dcb44abb8518ca967dcf0d35_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,754 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207119b18d6c094273a3e5fc5d18a6ec53_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207119b18d6c094273a3e5fc5d18a6ec53_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,755 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208e0bd79a8eca4198a09a13758dafa0b3_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208e0bd79a8eca4198a09a13758dafa0b3_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,757 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120909a65bb4c234273b0ead24d46d27750_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120909a65bb4c234273b0ead24d46d27750_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,760 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209b71ef305d834fb38261e0b240248d63_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209b71ef305d834fb38261e0b240248d63_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,763 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a0cfd6b3f78b42f2baec93bd83d31b1d_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a0cfd6b3f78b42f2baec93bd83d31b1d_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,764 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a3ee1453a0fd47f3bd251989ef1a5417_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a3ee1453a0fd47f3bd251989ef1a5417_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,766 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b2218fcf221c4c7fbb1b261d70f7e1a9_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b2218fcf221c4c7fbb1b261d70f7e1a9_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,767 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c5574409f38b4cffa0d5b60eb6620a3e_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c5574409f38b4cffa0d5b60eb6620a3e_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,769 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d01f59f7503940598bcc2691b25d6a01_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d01f59f7503940598bcc2691b25d6a01_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,770 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d3434ff01726422d95ac7db95a00defc_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d3434ff01726422d95ac7db95a00defc_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,771 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d6b0609f1af74621a5148809bf05dd6f_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d6b0609f1af74621a5148809bf05dd6f_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,773 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dd557dfe38e940ea89081fbe7466dfc7_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dd557dfe38e940ea89081fbe7466dfc7_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,774 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de43abbc75cf47d898e913510400c692_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de43abbc75cf47d898e913510400c692_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,776 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea7def61aa4643828e8d6dda39706fb3_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea7def61aa4643828e8d6dda39706fb3_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,777 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f1ec13efaf8244b5a5e2889383456553_ee28898c42e6b66b22f1f209861bee42 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f1ec13efaf8244b5a5e2889383456553_ee28898c42e6b66b22f1f209861bee42 2024-11-20T17:49:01,778 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T17:49:01,781 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,783 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T17:49:01,787 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T17:49:01,788 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,788 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T17:49:01,788 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732124941788"}]},"ts":"9223372036854775807"} 2024-11-20T17:49:01,791 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T17:49:01,791 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee28898c42e6b66b22f1f209861bee42, NAME => 'TestAcidGuarantees,,1732124907519.ee28898c42e6b66b22f1f209861bee42.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T17:49:01,791 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T17:49:01,791 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732124941791"}]},"ts":"9223372036854775807"} 2024-11-20T17:49:01,793 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T17:49:01,794 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-20T17:49:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T17:49:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T17:49:01,810 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:01,828 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-141067869_22 at /127.0.0.1:50354 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9110 (was 8483) - AvailableMemoryMB LEAK? - 2024-11-20T17:49:01,844 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=9109 2024-11-20T17:49:01,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:49:01,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:49:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:01,849 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:49:01,850 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:01,851 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:49:01,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-20T17:49:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T17:49:01,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742075_1251 (size=959) 2024-11-20T17:49:01,873 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:49:01,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742076_1252 (size=53) 2024-11-20T17:49:01,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:49:01,891 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee91b716d3c9cb4059bcd0deede2ec56, disabling compactions & flushes 2024-11-20T17:49:01,891 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:01,891 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:01,891 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. after waiting 0 ms 2024-11-20T17:49:01,891 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:01,891 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:01,891 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee91b716d3c9cb4059bcd0deede2ec56: Waiting for close lock at 1732124941891Disabling compacts and flushes for region at 1732124941891Disabling writes for close at 1732124941891Writing region close event to WAL at 1732124941891Closed at 1732124941891 2024-11-20T17:49:01,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:49:01,893 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732124941893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732124941893"}]},"ts":"1732124941893"} 2024-11-20T17:49:01,895 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:49:01,897 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:49:01,897 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124941897"}]},"ts":"1732124941897"} 2024-11-20T17:49:01,899 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T17:49:01,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, ASSIGN}] 2024-11-20T17:49:01,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, ASSIGN 2024-11-20T17:49:01,902 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, ASSIGN; state=OFFLINE, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=false 2024-11-20T17:49:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T17:49:02,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=ee91b716d3c9cb4059bcd0deede2ec56, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, ASSIGN because future has completed 2024-11-20T17:49:02,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:49:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T17:49:02,211 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,211 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:49:02,211 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,211 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:49:02,211 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,211 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,214 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,215 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee91b716d3c9cb4059bcd0deede2ec56 columnFamilyName A 2024-11-20T17:49:02,215 DEBUG [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:02,216 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] regionserver.HStore(327): Store=ee91b716d3c9cb4059bcd0deede2ec56/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:02,216 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,217 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee91b716d3c9cb4059bcd0deede2ec56 columnFamilyName B 2024-11-20T17:49:02,217 DEBUG [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:02,217 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] regionserver.HStore(327): Store=ee91b716d3c9cb4059bcd0deede2ec56/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:02,217 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,219 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee91b716d3c9cb4059bcd0deede2ec56 columnFamilyName C 2024-11-20T17:49:02,219 DEBUG [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:02,219 INFO [StoreOpener-ee91b716d3c9cb4059bcd0deede2ec56-1 {}] regionserver.HStore(327): Store=ee91b716d3c9cb4059bcd0deede2ec56/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:02,219 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,220 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,220 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,221 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,221 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,221 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:49:02,222 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,224 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:49:02,224 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened ee91b716d3c9cb4059bcd0deede2ec56; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71030431, jitterRate=0.05843590199947357}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:49:02,224 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,225 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for ee91b716d3c9cb4059bcd0deede2ec56: Running coprocessor pre-open hook at 1732124942211Writing region info on filesystem at 1732124942211Initializing all the Stores at 1732124942212 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124942212Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124942214 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124942214Cleaning up temporary data from old regions at 1732124942221 (+7 ms)Running coprocessor post-open hooks at 1732124942224 (+3 ms)Region opened successfully at 1732124942225 (+1 ms) 2024-11-20T17:49:02,225 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., pid=65, masterSystemTime=1732124942207 2024-11-20T17:49:02,227 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,227 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,228 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=ee91b716d3c9cb4059bcd0deede2ec56, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:49:02,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T17:49:02,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 in 174 msec 2024-11-20T17:49:02,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T17:49:02,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, ASSIGN in 332 msec 2024-11-20T17:49:02,233 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:49:02,233 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124942233"}]},"ts":"1732124942233"} 2024-11-20T17:49:02,234 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T17:49:02,235 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:49:02,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 389 msec 2024-11-20T17:49:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T17:49:02,488 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:02,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c08f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,490 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21dfa9c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,492 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4905445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,493 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37284d9b 2024-11-20T17:49:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:49:02,497 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:49:02,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd8e12d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb561cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,501 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9959c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429799b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,503 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b732cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cb52ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480421b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bcd897e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad68e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6327b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,510 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47eb695a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6bab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fdf952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,514 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664a88ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3fab70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0cbf10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f08fdc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,519 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57343f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735c732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c016790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,523 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f38d64a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212d94ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,526 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d9af43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,527 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330547f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bafd9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,529 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22510e95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2093c1c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b15c907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:02,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:02,533 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519d7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:02,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b657829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:02,536 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,536 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,536 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,536 DEBUG [Thread-1173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,537 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,539 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,540 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,540 DEBUG [Thread-1171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,545 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,549 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,551 DEBUG [Thread-1186 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:02,552 DEBUG [Thread-1186 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:02,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:02,553 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:02,554 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:02,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T17:49:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:02,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:49:02,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:02,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:02,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/0944e20c2c5643d79acdea830063310a is 50, key is test_row_0/A:col10/1732124942548/Put/seqid=0 2024-11-20T17:49:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125002592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125002595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125002597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125002599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125002602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742077_1253 (size=16681) 2024-11-20T17:49:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:02,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/0944e20c2c5643d79acdea830063310a 2024-11-20T17:49:02,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/6be40bdf97034519803ba1d0048fa865 is 50, key is test_row_0/B:col10/1732124942548/Put/seqid=0 2024-11-20T17:49:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125002707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125002718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125002718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125002718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125002718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742078_1254 (size=12001) 2024-11-20T17:49:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:02,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:02,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125002927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125002927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125002938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125002938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:02,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125002938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/6be40bdf97034519803ba1d0048fa865 2024-11-20T17:49:03,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/22c097b1df6640588e007f1af3701b2e is 50, key is test_row_0/C:col10/1732124942548/Put/seqid=0 2024-11-20T17:49:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125003237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125003238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742079_1255 (size=12001) 2024-11-20T17:49:03,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125003247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125003247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125003247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/22c097b1df6640588e007f1af3701b2e 2024-11-20T17:49:03,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/0944e20c2c5643d79acdea830063310a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0944e20c2c5643d79acdea830063310a 2024-11-20T17:49:03,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0944e20c2c5643d79acdea830063310a, entries=250, sequenceid=13, filesize=16.3 K 2024-11-20T17:49:03,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/6be40bdf97034519803ba1d0048fa865 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6be40bdf97034519803ba1d0048fa865 2024-11-20T17:49:03,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6be40bdf97034519803ba1d0048fa865, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:49:03,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/22c097b1df6640588e007f1af3701b2e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/22c097b1df6640588e007f1af3701b2e 2024-11-20T17:49:03,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/22c097b1df6640588e007f1af3701b2e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:49:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:03,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ee91b716d3c9cb4059bcd0deede2ec56 in 1135ms, sequenceid=13, compaction requested=false 2024-11-20T17:49:03,689 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T17:49:03,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:03,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:03,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125003767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125003767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125003765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125003769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125003772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/35ff967c1ef744598cead4e8a86af702 is 50, key is test_row_0/A:col10/1732124942590/Put/seqid=0 2024-11-20T17:49:03,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742080_1256 (size=14341) 2024-11-20T17:49:03,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/35ff967c1ef744598cead4e8a86af702 2024-11-20T17:49:03,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ce81974552ae421689d3acf3961e4fe9 is 50, key is test_row_0/B:col10/1732124942590/Put/seqid=0 2024-11-20T17:49:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125003886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:03,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742081_1257 (size=12001) 2024-11-20T17:49:03,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:03,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:03,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:03,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125004087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125004087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125004087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125004088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125004097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:04,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:04,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:04,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:04,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ce81974552ae421689d3acf3961e4fe9 2024-11-20T17:49:04,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/2512df9f1d494d748b12a7ac2fcad4e3 is 50, key is test_row_0/C:col10/1732124942590/Put/seqid=0 2024-11-20T17:49:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742082_1258 (size=12001) 2024-11-20T17:49:04,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/2512df9f1d494d748b12a7ac2fcad4e3 2024-11-20T17:49:04,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/35ff967c1ef744598cead4e8a86af702 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/35ff967c1ef744598cead4e8a86af702 2024-11-20T17:49:04,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/35ff967c1ef744598cead4e8a86af702, entries=200, sequenceid=39, filesize=14.0 K 2024-11-20T17:49:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ce81974552ae421689d3acf3961e4fe9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce81974552ae421689d3acf3961e4fe9 2024-11-20T17:49:04,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce81974552ae421689d3acf3961e4fe9, entries=150, sequenceid=39, filesize=11.7 K 2024-11-20T17:49:04,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/2512df9f1d494d748b12a7ac2fcad4e3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/2512df9f1d494d748b12a7ac2fcad4e3 2024-11-20T17:49:04,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/2512df9f1d494d748b12a7ac2fcad4e3, entries=150, sequenceid=39, filesize=11.7 K 2024-11-20T17:49:04,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ee91b716d3c9cb4059bcd0deede2ec56 in 582ms, sequenceid=39, compaction requested=false 2024-11-20T17:49:04,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:04,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:04,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:04,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/338feb8912714ca1bda468982ae946d0 is 50, key is test_row_1/A:col10/1732124944398/Put/seqid=0 2024-11-20T17:49:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742083_1259 (size=9657) 2024-11-20T17:49:04,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:04,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:04,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125004428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125004431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125004435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125004435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125004435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125004537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125004537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125004547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125004547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125004547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:04,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:04,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:04,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125004747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125004747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125004758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125004758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125004758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:04,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/338feb8912714ca1bda468982ae946d0 2024-11-20T17:49:04,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/2252a34433d74394a0820d131dde8107 is 50, key is test_row_1/B:col10/1732124944398/Put/seqid=0 2024-11-20T17:49:04,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742084_1260 (size=9657) 2024-11-20T17:49:04,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/2252a34433d74394a0820d131dde8107 2024-11-20T17:49:04,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/89af299303f143f7bf395c8d585a37ff is 50, key is test_row_1/C:col10/1732124944398/Put/seqid=0 2024-11-20T17:49:04,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742085_1261 (size=9657) 2024-11-20T17:49:04,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:04,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:04,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125005057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125005057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125005067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125005067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125005067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:05,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/89af299303f143f7bf395c8d585a37ff 2024-11-20T17:49:05,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/338feb8912714ca1bda468982ae946d0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/338feb8912714ca1bda468982ae946d0 2024-11-20T17:49:05,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/338feb8912714ca1bda468982ae946d0, entries=100, sequenceid=50, filesize=9.4 K 2024-11-20T17:49:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/2252a34433d74394a0820d131dde8107 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2252a34433d74394a0820d131dde8107 2024-11-20T17:49:05,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2252a34433d74394a0820d131dde8107, entries=100, sequenceid=50, filesize=9.4 K 2024-11-20T17:49:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/89af299303f143f7bf395c8d585a37ff as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/89af299303f143f7bf395c8d585a37ff 2024-11-20T17:49:05,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/89af299303f143f7bf395c8d585a37ff, entries=100, sequenceid=50, filesize=9.4 K 2024-11-20T17:49:05,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for ee91b716d3c9cb4059bcd0deede2ec56 in 921ms, sequenceid=50, compaction requested=true 2024-11-20T17:49:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:05,320 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:05,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:49:05,321 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:05,324 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:05,324 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:05,324 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,324 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6be40bdf97034519803ba1d0048fa865, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce81974552ae421689d3acf3961e4fe9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2252a34433d74394a0820d131dde8107] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=32.9 K 2024-11-20T17:49:05,325 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be40bdf97034519803ba1d0048fa865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732124942548 2024-11-20T17:49:05,326 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ce81974552ae421689d3acf3961e4fe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732124942590 2024-11-20T17:49:05,326 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2252a34433d74394a0820d131dde8107, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732124943765 2024-11-20T17:49:05,328 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:05,328 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:05,328 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,328 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0944e20c2c5643d79acdea830063310a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/35ff967c1ef744598cead4e8a86af702, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/338feb8912714ca1bda468982ae946d0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=39.7 K 2024-11-20T17:49:05,329 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0944e20c2c5643d79acdea830063310a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732124942546 2024-11-20T17:49:05,330 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ff967c1ef744598cead4e8a86af702, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732124942590 2024-11-20T17:49:05,331 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338feb8912714ca1bda468982ae946d0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732124943765 2024-11-20T17:49:05,341 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:05,341 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/4133315609da4cc5aebe5bc2e383a632 is 50, key is test_row_0/B:col10/1732124942590/Put/seqid=0 2024-11-20T17:49:05,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T17:49:05,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,342 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T17:49:05,350 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:05,351 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae is 50, key is test_row_0/A:col10/1732124942590/Put/seqid=0 2024-11-20T17:49:05,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f1c0469465f94162b0576f5d05ff2dfa is 50, key is test_row_0/A:col10/1732124944434/Put/seqid=0 2024-11-20T17:49:05,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742086_1262 (size=12104) 2024-11-20T17:49:05,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742087_1263 (size=12104) 2024-11-20T17:49:05,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742088_1264 (size=12001) 2024-11-20T17:49:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:05,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125005577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125005577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125005577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125005580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125005581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125005687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125005687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,806 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/4133315609da4cc5aebe5bc2e383a632 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/4133315609da4cc5aebe5bc2e383a632 2024-11-20T17:49:05,813 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 4133315609da4cc5aebe5bc2e383a632(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:05,813 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:05,813 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=13, startTime=1732124945320; duration=0sec 2024-11-20T17:49:05,813 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:05,813 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:05,813 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:05,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:05,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:05,815 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:05,815 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/22c097b1df6640588e007f1af3701b2e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/2512df9f1d494d748b12a7ac2fcad4e3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/89af299303f143f7bf395c8d585a37ff] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=32.9 K 2024-11-20T17:49:05,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c097b1df6640588e007f1af3701b2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732124942548 2024-11-20T17:49:05,816 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2512df9f1d494d748b12a7ac2fcad4e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732124942590 2024-11-20T17:49:05,816 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 89af299303f143f7bf395c8d585a37ff, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732124943765 2024-11-20T17:49:05,823 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:05,824 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/f52611722c8247b9abe07af419a2ae78 is 50, key is test_row_0/C:col10/1732124942590/Put/seqid=0 2024-11-20T17:49:05,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742089_1265 (size=12104) 2024-11-20T17:49:05,843 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/f52611722c8247b9abe07af419a2ae78 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f52611722c8247b9abe07af419a2ae78 2024-11-20T17:49:05,853 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into f52611722c8247b9abe07af419a2ae78(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:05,853 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:05,853 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=13, startTime=1732124945321; duration=0sec 2024-11-20T17:49:05,853 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:05,853 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:05,862 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f1c0469465f94162b0576f5d05ff2dfa 2024-11-20T17:49:05,881 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae 2024-11-20T17:49:05,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/e1e089a2bb154e1d9ec50993a003775b is 50, key is test_row_0/B:col10/1732124944434/Put/seqid=0 2024-11-20T17:49:05,888 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into 2aa8ea6f30f14ed7824cfc0c9f2ffcae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:05,888 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:05,888 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=13, startTime=1732124945320; duration=0sec 2024-11-20T17:49:05,889 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:05,889 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125005897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125005897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:05,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742090_1266 (size=12001) 2024-11-20T17:49:05,926 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/e1e089a2bb154e1d9ec50993a003775b 2024-11-20T17:49:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/fa3fb2b67ae748749b6edd332d0c0097 is 50, key is test_row_0/C:col10/1732124944434/Put/seqid=0 2024-11-20T17:49:05,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742091_1267 (size=12001) 2024-11-20T17:49:05,971 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/fa3fb2b67ae748749b6edd332d0c0097 2024-11-20T17:49:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f1c0469465f94162b0576f5d05ff2dfa as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f1c0469465f94162b0576f5d05ff2dfa 2024-11-20T17:49:05,992 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f1c0469465f94162b0576f5d05ff2dfa, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T17:49:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/e1e089a2bb154e1d9ec50993a003775b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/e1e089a2bb154e1d9ec50993a003775b 2024-11-20T17:49:06,002 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/e1e089a2bb154e1d9ec50993a003775b, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T17:49:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/fa3fb2b67ae748749b6edd332d0c0097 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fa3fb2b67ae748749b6edd332d0c0097 2024-11-20T17:49:06,008 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fa3fb2b67ae748749b6edd332d0c0097, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T17:49:06,009 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ee91b716d3c9cb4059bcd0deede2ec56 in 667ms, sequenceid=77, compaction requested=false 2024-11-20T17:49:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:06,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T17:49:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T17:49:06,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T17:49:06,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4500 sec 2024-11-20T17:49:06,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 3.4620 sec 2024-11-20T17:49:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:06,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:06,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:06,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/a07ce84d28034f69b198fae03f24749d is 50, key is test_row_0/A:col10/1732124946207/Put/seqid=0 2024-11-20T17:49:06,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742092_1268 (size=12001) 2024-11-20T17:49:06,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/a07ce84d28034f69b198fae03f24749d 2024-11-20T17:49:06,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/9e930ae201ce42d398b4ddea0814885e is 50, key is test_row_0/B:col10/1732124946207/Put/seqid=0 2024-11-20T17:49:06,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742093_1269 (size=12001) 2024-11-20T17:49:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125006269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125006270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125006377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125006377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125006587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125006587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125006587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125006587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,591 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,591 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125006597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,601 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/9e930ae201ce42d398b4ddea0814885e 2024-11-20T17:49:06,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/8252587a3fbe41d3b79dbc18e3770bc9 is 50, key is test_row_0/C:col10/1732124946207/Put/seqid=0 2024-11-20T17:49:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742094_1270 (size=12001) 2024-11-20T17:49:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T17:49:06,728 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:06,728 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T17:49:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T17:49:06,730 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:06,730 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:06,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T17:49:06,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T17:49:06,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:06,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:06,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:06,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T17:49:07,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:07,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:07,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T17:49:07,064 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:49:07,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/8252587a3fbe41d3b79dbc18e3770bc9 2024-11-20T17:49:07,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/a07ce84d28034f69b198fae03f24749d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/a07ce84d28034f69b198fae03f24749d 2024-11-20T17:49:07,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/a07ce84d28034f69b198fae03f24749d, entries=150, sequenceid=91, filesize=11.7 K 2024-11-20T17:49:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/9e930ae201ce42d398b4ddea0814885e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9e930ae201ce42d398b4ddea0814885e 2024-11-20T17:49:07,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9e930ae201ce42d398b4ddea0814885e, entries=150, sequenceid=91, filesize=11.7 K 2024-11-20T17:49:07,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/8252587a3fbe41d3b79dbc18e3770bc9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/8252587a3fbe41d3b79dbc18e3770bc9 2024-11-20T17:49:07,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/8252587a3fbe41d3b79dbc18e3770bc9, entries=150, sequenceid=91, filesize=11.7 K 2024-11-20T17:49:07,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee91b716d3c9cb4059bcd0deede2ec56 in 867ms, sequenceid=91, compaction requested=true 2024-11-20T17:49:07,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:07,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:07,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:07,088 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:07,088 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:07,090 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:07,090 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:07,090 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,090 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/4133315609da4cc5aebe5bc2e383a632, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/e1e089a2bb154e1d9ec50993a003775b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9e930ae201ce42d398b4ddea0814885e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=35.3 K 2024-11-20T17:49:07,090 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:07,090 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:07,091 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,091 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f1c0469465f94162b0576f5d05ff2dfa, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/a07ce84d28034f69b198fae03f24749d] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=35.3 K 2024-11-20T17:49:07,091 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4133315609da4cc5aebe5bc2e383a632, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732124942590 2024-11-20T17:49:07,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:07,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:07,091 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa8ea6f30f14ed7824cfc0c9f2ffcae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732124942590 2024-11-20T17:49:07,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:07,092 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e089a2bb154e1d9ec50993a003775b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732124944433 2024-11-20T17:49:07,092 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1c0469465f94162b0576f5d05ff2dfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732124944433 2024-11-20T17:49:07,092 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e930ae201ce42d398b4ddea0814885e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732124945568 2024-11-20T17:49:07,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:07,092 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07ce84d28034f69b198fae03f24749d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732124945568 2024-11-20T17:49:07,100 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:07,101 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/086bdf1421b44aa282c46396c262a794 is 50, key is test_row_0/A:col10/1732124946207/Put/seqid=0 2024-11-20T17:49:07,101 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:07,101 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/d53d3e7b6cb64b438cf064eb4013724c is 50, key is test_row_0/B:col10/1732124946207/Put/seqid=0 2024-11-20T17:49:07,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742095_1271 (size=12207) 2024-11-20T17:49:07,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742096_1272 (size=12207) 2024-11-20T17:49:07,122 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/086bdf1421b44aa282c46396c262a794 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/086bdf1421b44aa282c46396c262a794 2024-11-20T17:49:07,142 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into 086bdf1421b44aa282c46396c262a794(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:07,142 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:07,142 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=13, startTime=1732124947088; duration=0sec 2024-11-20T17:49:07,142 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:07,142 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:07,142 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:07,144 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:07,144 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:07,145 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,145 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f52611722c8247b9abe07af419a2ae78, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fa3fb2b67ae748749b6edd332d0c0097, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/8252587a3fbe41d3b79dbc18e3770bc9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=35.3 K 2024-11-20T17:49:07,145 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52611722c8247b9abe07af419a2ae78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732124942590 2024-11-20T17:49:07,145 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa3fb2b67ae748749b6edd332d0c0097, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732124944433 2024-11-20T17:49:07,145 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8252587a3fbe41d3b79dbc18e3770bc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732124945568 2024-11-20T17:49:07,160 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:07,160 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/d52aa2764d6b4fb5bc59b7e6bf5170a7 is 50, key is test_row_0/C:col10/1732124946207/Put/seqid=0 2024-11-20T17:49:07,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742097_1273 (size=12207) 2024-11-20T17:49:07,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T17:49:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,188 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:07,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/e012921b4156499bb5a18813eef90357 is 50, key is test_row_0/A:col10/1732124946267/Put/seqid=0 2024-11-20T17:49:07,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742098_1274 (size=12001) 2024-11-20T17:49:07,196 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/e012921b4156499bb5a18813eef90357 2024-11-20T17:49:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ce77f6def5344d1f9669b808de4d8c93 is 50, key is test_row_0/B:col10/1732124946267/Put/seqid=0 2024-11-20T17:49:07,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742099_1275 (size=12001) 2024-11-20T17:49:07,210 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ce77f6def5344d1f9669b808de4d8c93 2024-11-20T17:49:07,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/5bad65c1018b42ea8f22adb67cda1be4 is 50, key is test_row_0/C:col10/1732124946267/Put/seqid=0 2024-11-20T17:49:07,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742100_1276 (size=12001) 2024-11-20T17:49:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T17:49:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:07,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:07,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125007422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125007423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,517 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/d53d3e7b6cb64b438cf064eb4013724c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/d53d3e7b6cb64b438cf064eb4013724c 2024-11-20T17:49:07,529 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into d53d3e7b6cb64b438cf064eb4013724c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:07,529 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:07,529 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=13, startTime=1732124947088; duration=0sec 2024-11-20T17:49:07,529 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:07,529 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:07,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125007527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125007537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,587 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/d52aa2764d6b4fb5bc59b7e6bf5170a7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/d52aa2764d6b4fb5bc59b7e6bf5170a7 2024-11-20T17:49:07,600 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into d52aa2764d6b4fb5bc59b7e6bf5170a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:07,600 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:07,600 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=13, startTime=1732124947091; duration=0sec 2024-11-20T17:49:07,601 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:07,601 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:07,622 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/5bad65c1018b42ea8f22adb67cda1be4 2024-11-20T17:49:07,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/e012921b4156499bb5a18813eef90357 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/e012921b4156499bb5a18813eef90357 2024-11-20T17:49:07,635 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/e012921b4156499bb5a18813eef90357, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T17:49:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ce77f6def5344d1f9669b808de4d8c93 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce77f6def5344d1f9669b808de4d8c93 2024-11-20T17:49:07,640 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce77f6def5344d1f9669b808de4d8c93, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T17:49:07,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/5bad65c1018b42ea8f22adb67cda1be4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/5bad65c1018b42ea8f22adb67cda1be4 2024-11-20T17:49:07,649 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/5bad65c1018b42ea8f22adb67cda1be4, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T17:49:07,650 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee91b716d3c9cb4059bcd0deede2ec56 in 462ms, sequenceid=117, compaction requested=false 2024-11-20T17:49:07,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:07,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T17:49:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T17:49:07,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T17:49:07,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 922 msec 2024-11-20T17:49:07,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 926 msec 2024-11-20T17:49:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:07,739 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:07,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:49:07,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d3a961bd91004d0085af2cb168039eed is 50, key is test_row_1/A:col10/1732124947422/Put/seqid=0 2024-11-20T17:49:07,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742101_1277 (size=9707) 2024-11-20T17:49:07,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d3a961bd91004d0085af2cb168039eed 2024-11-20T17:49:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125007802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125007805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ba2fb363d0c444d1bb75907ee6e06be2 is 50, key is test_row_1/B:col10/1732124947422/Put/seqid=0 2024-11-20T17:49:07,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742102_1278 (size=9707) 2024-11-20T17:49:07,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ba2fb363d0c444d1bb75907ee6e06be2 2024-11-20T17:49:07,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/dd673359af2f4e0bbfca9004cd3a0148 is 50, key is test_row_1/C:col10/1732124947422/Put/seqid=0 2024-11-20T17:49:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742103_1279 (size=9707) 2024-11-20T17:49:07,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/dd673359af2f4e0bbfca9004cd3a0148 2024-11-20T17:49:07,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d3a961bd91004d0085af2cb168039eed as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d3a961bd91004d0085af2cb168039eed 2024-11-20T17:49:07,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d3a961bd91004d0085af2cb168039eed, entries=100, sequenceid=131, filesize=9.5 K 2024-11-20T17:49:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/ba2fb363d0c444d1bb75907ee6e06be2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ba2fb363d0c444d1bb75907ee6e06be2 2024-11-20T17:49:07,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ba2fb363d0c444d1bb75907ee6e06be2, entries=100, sequenceid=131, filesize=9.5 K 2024-11-20T17:49:07,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/dd673359af2f4e0bbfca9004cd3a0148 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/dd673359af2f4e0bbfca9004cd3a0148 2024-11-20T17:49:07,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/dd673359af2f4e0bbfca9004cd3a0148, entries=100, sequenceid=131, filesize=9.5 K 2024-11-20T17:49:07,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ee91b716d3c9cb4059bcd0deede2ec56 in 123ms, sequenceid=131, compaction requested=true 2024-11-20T17:49:07,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:07,862 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:07,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:07,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:07,863 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:07,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:07,863 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:07,863 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:07,864 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:07,864 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:07,864 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,864 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/d53d3e7b6cb64b438cf064eb4013724c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce77f6def5344d1f9669b808de4d8c93, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ba2fb363d0c444d1bb75907ee6e06be2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=33.1 K 2024-11-20T17:49:07,864 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:07,864 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/086bdf1421b44aa282c46396c262a794, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/e012921b4156499bb5a18813eef90357, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d3a961bd91004d0085af2cb168039eed] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=33.1 K 2024-11-20T17:49:07,864 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d53d3e7b6cb64b438cf064eb4013724c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732124945568 2024-11-20T17:49:07,864 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086bdf1421b44aa282c46396c262a794, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732124945568 2024-11-20T17:49:07,865 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ce77f6def5344d1f9669b808de4d8c93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732124946263 2024-11-20T17:49:07,865 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e012921b4156499bb5a18813eef90357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732124946263 2024-11-20T17:49:07,865 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ba2fb363d0c444d1bb75907ee6e06be2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732124947422 2024-11-20T17:49:07,865 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a961bd91004d0085af2cb168039eed, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732124947422 2024-11-20T17:49:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T17:49:07,868 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:07,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T17:49:07,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T17:49:07,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:07,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:07,875 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:07,876 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/731c4e2760714577b7129ffb860386f8 is 50, key is test_row_0/B:col10/1732124946267/Put/seqid=0 2024-11-20T17:49:07,890 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:07,890 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/b9f838f493db448d8cde842e7dd42eea is 50, key is test_row_0/A:col10/1732124946267/Put/seqid=0 2024-11-20T17:49:07,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742104_1280 (size=12359) 2024-11-20T17:49:07,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742105_1281 (size=12359) 2024-11-20T17:49:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:07,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:07,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:07,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f936ee51c5134c79bf2bb71c8f580a3e is 50, key is test_row_0/A:col10/1732124947803/Put/seqid=0 2024-11-20T17:49:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742106_1282 (size=14541) 2024-11-20T17:49:07,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f936ee51c5134c79bf2bb71c8f580a3e 2024-11-20T17:49:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125007935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125007937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:07,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3d99d4ac0ebd47238ed61d0c11dc0ccf is 50, key is test_row_0/B:col10/1732124947803/Put/seqid=0 2024-11-20T17:49:07,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742107_1283 (size=12151) 2024-11-20T17:49:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T17:49:08,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T17:49:08,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:08,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125008047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125008047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T17:49:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:08,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T17:49:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125008257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125008258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,306 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/731c4e2760714577b7129ffb860386f8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/731c4e2760714577b7129ffb860386f8 2024-11-20T17:49:08,313 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/b9f838f493db448d8cde842e7dd42eea as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/b9f838f493db448d8cde842e7dd42eea 2024-11-20T17:49:08,315 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 731c4e2760714577b7129ffb860386f8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:08,315 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:08,315 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=13, startTime=1732124947862; duration=0sec 2024-11-20T17:49:08,315 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:08,315 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:08,315 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:08,317 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:08,317 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:08,317 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,318 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/d52aa2764d6b4fb5bc59b7e6bf5170a7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/5bad65c1018b42ea8f22adb67cda1be4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/dd673359af2f4e0bbfca9004cd3a0148] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=33.1 K 2024-11-20T17:49:08,320 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into b9f838f493db448d8cde842e7dd42eea(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:08,320 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:08,320 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=13, startTime=1732124947862; duration=0sec 2024-11-20T17:49:08,320 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:08,320 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:08,320 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d52aa2764d6b4fb5bc59b7e6bf5170a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732124945568 2024-11-20T17:49:08,324 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bad65c1018b42ea8f22adb67cda1be4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732124946263 2024-11-20T17:49:08,325 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting dd673359af2f4e0bbfca9004cd3a0148, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732124947422 2024-11-20T17:49:08,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T17:49:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,339 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:08,339 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/4ce6dbb7e8ca4240a02222ffd60f71e5 is 50, key is test_row_0/C:col10/1732124946267/Put/seqid=0 2024-11-20T17:49:08,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3d99d4ac0ebd47238ed61d0c11dc0ccf 2024-11-20T17:49:08,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742108_1284 (size=12359) 2024-11-20T17:49:08,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/11ff9620637f412b9aeefa57734168f1 is 50, key is test_row_0/C:col10/1732124947803/Put/seqid=0 2024-11-20T17:49:08,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742109_1285 (size=12151) 2024-11-20T17:49:08,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T17:49:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:08,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T17:49:08,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125008568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125008568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125008607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,609 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125008617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125008618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:08,620 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:08,620 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:08,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T17:49:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:08,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:08,764 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/4ce6dbb7e8ca4240a02222ffd60f71e5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/4ce6dbb7e8ca4240a02222ffd60f71e5 2024-11-20T17:49:08,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/11ff9620637f412b9aeefa57734168f1 2024-11-20T17:49:08,769 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into 4ce6dbb7e8ca4240a02222ffd60f71e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:08,769 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:08,769 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=13, startTime=1732124947863; duration=0sec 2024-11-20T17:49:08,769 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:08,769 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:08,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f936ee51c5134c79bf2bb71c8f580a3e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f936ee51c5134c79bf2bb71c8f580a3e 2024-11-20T17:49:08,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f936ee51c5134c79bf2bb71c8f580a3e, entries=200, sequenceid=157, filesize=14.2 K 2024-11-20T17:49:08,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3d99d4ac0ebd47238ed61d0c11dc0ccf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3d99d4ac0ebd47238ed61d0c11dc0ccf 2024-11-20T17:49:08,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3d99d4ac0ebd47238ed61d0c11dc0ccf, entries=150, sequenceid=157, filesize=11.9 K 2024-11-20T17:49:08,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/11ff9620637f412b9aeefa57734168f1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/11ff9620637f412b9aeefa57734168f1 2024-11-20T17:49:08,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/11ff9620637f412b9aeefa57734168f1, entries=150, sequenceid=157, filesize=11.9 K 2024-11-20T17:49:08,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ee91b716d3c9cb4059bcd0deede2ec56 in 862ms, sequenceid=157, compaction requested=false 2024-11-20T17:49:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:08,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T17:49:08,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,796 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T17:49:08,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5017fbb663094ea69e2257855e2ae6b9 is 50, key is test_row_0/A:col10/1732124947925/Put/seqid=0 2024-11-20T17:49:08,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742110_1286 (size=12151) 2024-11-20T17:49:08,818 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5017fbb663094ea69e2257855e2ae6b9 2024-11-20T17:49:08,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/a13d6e317dd840df914bed60428ad16e is 50, key is test_row_0/B:col10/1732124947925/Put/seqid=0 2024-11-20T17:49:08,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742111_1287 (size=12151) 2024-11-20T17:49:08,855 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/a13d6e317dd840df914bed60428ad16e 2024-11-20T17:49:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/ebc5352d716b47f2acdb2d90c5847c18 is 50, key is test_row_0/C:col10/1732124947925/Put/seqid=0 2024-11-20T17:49:08,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742112_1288 (size=12151) 2024-11-20T17:49:08,875 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/ebc5352d716b47f2acdb2d90c5847c18 2024-11-20T17:49:08,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5017fbb663094ea69e2257855e2ae6b9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5017fbb663094ea69e2257855e2ae6b9 2024-11-20T17:49:08,886 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5017fbb663094ea69e2257855e2ae6b9, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T17:49:08,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/a13d6e317dd840df914bed60428ad16e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a13d6e317dd840df914bed60428ad16e 2024-11-20T17:49:08,896 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a13d6e317dd840df914bed60428ad16e, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T17:49:08,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/ebc5352d716b47f2acdb2d90c5847c18 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/ebc5352d716b47f2acdb2d90c5847c18 2024-11-20T17:49:08,901 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/ebc5352d716b47f2acdb2d90c5847c18, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T17:49:08,902 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for ee91b716d3c9cb4059bcd0deede2ec56 in 106ms, sequenceid=170, compaction requested=true 2024-11-20T17:49:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:08,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T17:49:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T17:49:08,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T17:49:08,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-20T17:49:08,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0520 sec 2024-11-20T17:49:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T17:49:09,007 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:09,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T17:49:09,010 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T17:49:09,011 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:09,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:09,092 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:09,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:09,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/1e0f1f9a09ac48559fce19fbe8a5b5d9 is 50, key is test_row_0/A:col10/1732124949085/Put/seqid=0 2024-11-20T17:49:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T17:49:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742113_1289 (size=16931) 2024-11-20T17:49:09,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125009160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125009163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125009267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125009277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T17:49:09,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125009477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125009487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/1e0f1f9a09ac48559fce19fbe8a5b5d9 2024-11-20T17:49:09,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/71d0371f689d4c12a1601d69208feedd is 50, key is test_row_0/B:col10/1732124949085/Put/seqid=0 2024-11-20T17:49:09,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742114_1290 (size=12151) 2024-11-20T17:49:09,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T17:49:09,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125009787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125009797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:09,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:09,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:09,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:09,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:09,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/71d0371f689d4c12a1601d69208feedd 2024-11-20T17:49:09,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/7534e84a14bd49edb2dfd217fc6d95bb is 50, key is test_row_0/C:col10/1732124949085/Put/seqid=0 2024-11-20T17:49:09,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742115_1291 (size=12151) 2024-11-20T17:49:10,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:10,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:10,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T17:49:10,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:10,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125010297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:10,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125010307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:10,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/7534e84a14bd49edb2dfd217fc6d95bb 2024-11-20T17:49:10,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/1e0f1f9a09ac48559fce19fbe8a5b5d9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1e0f1f9a09ac48559fce19fbe8a5b5d9 2024-11-20T17:49:10,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1e0f1f9a09ac48559fce19fbe8a5b5d9, entries=250, sequenceid=181, filesize=16.5 K 2024-11-20T17:49:10,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/71d0371f689d4c12a1601d69208feedd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/71d0371f689d4c12a1601d69208feedd 2024-11-20T17:49:10,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/71d0371f689d4c12a1601d69208feedd, entries=150, sequenceid=181, filesize=11.9 K 2024-11-20T17:49:10,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/7534e84a14bd49edb2dfd217fc6d95bb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/7534e84a14bd49edb2dfd217fc6d95bb 2024-11-20T17:49:10,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/7534e84a14bd49edb2dfd217fc6d95bb, entries=150, sequenceid=181, filesize=11.9 K 2024-11-20T17:49:10,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee91b716d3c9cb4059bcd0deede2ec56 in 1290ms, sequenceid=181, compaction requested=true 2024-11-20T17:49:10,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:10,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:10,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:10,382 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:10,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:10,383 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:10,384 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:10,384 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55982 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:10,384 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:10,384 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:10,384 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,384 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,384 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/731c4e2760714577b7129ffb860386f8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3d99d4ac0ebd47238ed61d0c11dc0ccf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a13d6e317dd840df914bed60428ad16e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/71d0371f689d4c12a1601d69208feedd] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=47.7 K 2024-11-20T17:49:10,384 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/b9f838f493db448d8cde842e7dd42eea, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f936ee51c5134c79bf2bb71c8f580a3e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5017fbb663094ea69e2257855e2ae6b9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1e0f1f9a09ac48559fce19fbe8a5b5d9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=54.7 K 2024-11-20T17:49:10,385 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 731c4e2760714577b7129ffb860386f8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732124946267 2024-11-20T17:49:10,385 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9f838f493db448d8cde842e7dd42eea, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732124946267 2024-11-20T17:49:10,385 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f936ee51c5134c79bf2bb71c8f580a3e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124947796 2024-11-20T17:49:10,385 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d99d4ac0ebd47238ed61d0c11dc0ccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124947803 2024-11-20T17:49:10,385 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5017fbb663094ea69e2257855e2ae6b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732124947925 2024-11-20T17:49:10,386 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a13d6e317dd840df914bed60428ad16e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732124947925 2024-11-20T17:49:10,386 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e0f1f9a09ac48559fce19fbe8a5b5d9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732124949080 2024-11-20T17:49:10,386 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d0371f689d4c12a1601d69208feedd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732124949080 2024-11-20T17:49:10,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T17:49:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,393 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d654ff20a29246be8a07c2122a0e1a83 is 50, key is test_row_0/A:col10/1732124949161/Put/seqid=0 2024-11-20T17:49:10,412 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#243 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:10,413 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/fe726db31d5145d89510be8f2ddb5190 is 50, key is test_row_0/A:col10/1732124949085/Put/seqid=0 2024-11-20T17:49:10,426 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:10,426 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/7db57801e8b64e7cb0bb4ed336879f25 is 50, key is test_row_0/B:col10/1732124949085/Put/seqid=0 2024-11-20T17:49:10,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742116_1292 (size=12151) 2024-11-20T17:49:10,442 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d654ff20a29246be8a07c2122a0e1a83 2024-11-20T17:49:10,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/91c82e65443d45089e4fbc33173dc0b3 is 50, key is test_row_0/B:col10/1732124949161/Put/seqid=0 2024-11-20T17:49:10,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742117_1293 (size=12595) 2024-11-20T17:49:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742119_1295 (size=12151) 2024-11-20T17:49:10,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742118_1294 (size=12595) 2024-11-20T17:49:10,482 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/91c82e65443d45089e4fbc33173dc0b3 2024-11-20T17:49:10,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/08ec618d6ff54cbea90c47e21f148e49 is 50, key is test_row_0/C:col10/1732124949161/Put/seqid=0 2024-11-20T17:49:10,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742120_1296 (size=12151) 2024-11-20T17:49:10,865 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/fe726db31d5145d89510be8f2ddb5190 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/fe726db31d5145d89510be8f2ddb5190 2024-11-20T17:49:10,870 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into fe726db31d5145d89510be8f2ddb5190(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:10,870 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:10,870 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=12, startTime=1732124950382; duration=0sec 2024-11-20T17:49:10,870 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:10,870 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:10,870 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:10,872 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:10,872 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:10,872 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,872 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/4ce6dbb7e8ca4240a02222ffd60f71e5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/11ff9620637f412b9aeefa57734168f1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/ebc5352d716b47f2acdb2d90c5847c18, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/7534e84a14bd49edb2dfd217fc6d95bb] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=47.7 K 2024-11-20T17:49:10,872 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce6dbb7e8ca4240a02222ffd60f71e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732124946267 2024-11-20T17:49:10,873 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ff9620637f412b9aeefa57734168f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124947803 2024-11-20T17:49:10,873 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc5352d716b47f2acdb2d90c5847c18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732124947925 2024-11-20T17:49:10,873 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7534e84a14bd49edb2dfd217fc6d95bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732124949080 2024-11-20T17:49:10,880 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:10,880 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/549f140ffa974633b6b5f76417ab392e is 50, key is test_row_0/C:col10/1732124949085/Put/seqid=0 2024-11-20T17:49:10,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742121_1297 (size=12595) 2024-11-20T17:49:10,889 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/7db57801e8b64e7cb0bb4ed336879f25 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/7db57801e8b64e7cb0bb4ed336879f25 2024-11-20T17:49:10,901 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 7db57801e8b64e7cb0bb4ed336879f25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:10,901 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:10,901 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=12, startTime=1732124950382; duration=0sec 2024-11-20T17:49:10,901 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:10,901 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:10,907 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/08ec618d6ff54cbea90c47e21f148e49 2024-11-20T17:49:10,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d654ff20a29246be8a07c2122a0e1a83 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d654ff20a29246be8a07c2122a0e1a83 2024-11-20T17:49:10,917 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d654ff20a29246be8a07c2122a0e1a83, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T17:49:10,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/91c82e65443d45089e4fbc33173dc0b3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/91c82e65443d45089e4fbc33173dc0b3 2024-11-20T17:49:10,923 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/91c82e65443d45089e4fbc33173dc0b3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T17:49:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/08ec618d6ff54cbea90c47e21f148e49 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/08ec618d6ff54cbea90c47e21f148e49 2024-11-20T17:49:10,928 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/08ec618d6ff54cbea90c47e21f148e49, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T17:49:10,929 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for ee91b716d3c9cb4059bcd0deede2ec56 in 536ms, sequenceid=206, compaction requested=false 2024-11-20T17:49:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T17:49:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T17:49:10,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T17:49:10,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9200 sec 2024-11-20T17:49:10,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9260 sec 2024-11-20T17:49:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T17:49:11,157 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:11,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T17:49:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T17:49:11,160 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:11,160 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:11,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T17:49:11,294 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/549f140ffa974633b6b5f76417ab392e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/549f140ffa974633b6b5f76417ab392e 2024-11-20T17:49:11,299 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into 549f140ffa974633b6b5f76417ab392e(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:11,299 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:11,299 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=12, startTime=1732124950383; duration=0sec 2024-11-20T17:49:11,299 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:11,299 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:11,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T17:49:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:11,313 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-20T17:49:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/0a7146134d9e42cdbbeea19708eb753c is 50, key is test_row_0/A:col10/1732124951313/Put/seqid=0 2024-11-20T17:49:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:11,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742122_1298 (size=9757) 2024-11-20T17:49:11,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125011371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:11,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125011375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T17:49:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125011477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:11,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125011487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:11,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125011687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:11,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125011697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:11,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:11,735 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/0a7146134d9e42cdbbeea19708eb753c 2024-11-20T17:49:11,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/2c3f3446bc8d43b3836f33546517789b is 50, key is test_row_0/B:col10/1732124951313/Put/seqid=0 2024-11-20T17:49:11,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742123_1299 (size=9757) 2024-11-20T17:49:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T17:49:11,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125011997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125012007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,154 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/2c3f3446bc8d43b3836f33546517789b 2024-11-20T17:49:12,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/412ee5258fc14ba99ffafa643d2e6647 is 50, key is test_row_0/C:col10/1732124951313/Put/seqid=0 2024-11-20T17:49:12,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742124_1300 (size=9757) 2024-11-20T17:49:12,164 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/412ee5258fc14ba99ffafa643d2e6647 2024-11-20T17:49:12,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/0a7146134d9e42cdbbeea19708eb753c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0a7146134d9e42cdbbeea19708eb753c 2024-11-20T17:49:12,173 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0a7146134d9e42cdbbeea19708eb753c, entries=100, sequenceid=215, filesize=9.5 K 2024-11-20T17:49:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/2c3f3446bc8d43b3836f33546517789b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2c3f3446bc8d43b3836f33546517789b 2024-11-20T17:49:12,182 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2c3f3446bc8d43b3836f33546517789b, entries=100, sequenceid=215, filesize=9.5 K 2024-11-20T17:49:12,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/412ee5258fc14ba99ffafa643d2e6647 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/412ee5258fc14ba99ffafa643d2e6647 2024-11-20T17:49:12,197 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/412ee5258fc14ba99ffafa643d2e6647, entries=100, sequenceid=215, filesize=9.5 K 2024-11-20T17:49:12,199 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for ee91b716d3c9cb4059bcd0deede2ec56 in 885ms, sequenceid=215, compaction requested=true 2024-11-20T17:49:12,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:12,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:12,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T17:49:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T17:49:12,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T17:49:12,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-20T17:49:12,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0450 sec 2024-11-20T17:49:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T17:49:12,297 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:12,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T17:49:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T17:49:12,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:12,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:12,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T17:49:12,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T17:49:12,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:12,455 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-20T17:49:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/36edd9d8d7384a67a40be0e93dc38e41 is 50, key is test_row_0/A:col10/1732124951374/Put/seqid=0 2024-11-20T17:49:12,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742125_1301 (size=12151) 2024-11-20T17:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:12,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125012517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125012518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T17:49:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125012617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,620 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125012627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125012637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,639 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125012667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,672 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125012837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:12,885 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/36edd9d8d7384a67a40be0e93dc38e41 2024-11-20T17:49:12,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/6fae00262df140329aa351f6081c66c4 is 50, key is test_row_0/B:col10/1732124951374/Put/seqid=0 2024-11-20T17:49:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742126_1302 (size=12151) 2024-11-20T17:49:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T17:49:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125013147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:13,298 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/6fae00262df140329aa351f6081c66c4 2024-11-20T17:49:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/44bf9b52003744db8626a26ef65e8c1a is 50, key is test_row_0/C:col10/1732124951374/Put/seqid=0 2024-11-20T17:49:13,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742127_1303 (size=12151) 2024-11-20T17:49:13,314 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/44bf9b52003744db8626a26ef65e8c1a 2024-11-20T17:49:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/36edd9d8d7384a67a40be0e93dc38e41 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/36edd9d8d7384a67a40be0e93dc38e41 2024-11-20T17:49:13,323 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/36edd9d8d7384a67a40be0e93dc38e41, entries=150, sequenceid=245, filesize=11.9 K 2024-11-20T17:49:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/6fae00262df140329aa351f6081c66c4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6fae00262df140329aa351f6081c66c4 2024-11-20T17:49:13,328 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6fae00262df140329aa351f6081c66c4, entries=150, sequenceid=245, filesize=11.9 K 2024-11-20T17:49:13,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/44bf9b52003744db8626a26ef65e8c1a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/44bf9b52003744db8626a26ef65e8c1a 2024-11-20T17:49:13,333 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/44bf9b52003744db8626a26ef65e8c1a, entries=150, sequenceid=245, filesize=11.9 K 2024-11-20T17:49:13,334 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for ee91b716d3c9cb4059bcd0deede2ec56 in 879ms, sequenceid=245, compaction requested=true 2024-11-20T17:49:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T17:49:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T17:49:13,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T17:49:13,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-11-20T17:49:13,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0400 sec 2024-11-20T17:49:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T17:49:13,437 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:13,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-20T17:49:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T17:49:13,440 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:13,440 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:13,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:13,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:13,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T17:49:13,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/62c58e98b9a1488ca6b7055dc865fb13 is 50, key is test_row_0/A:col10/1732124953540/Put/seqid=0 2024-11-20T17:49:13,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742128_1304 (size=14541) 2024-11-20T17:49:13,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/62c58e98b9a1488ca6b7055dc865fb13 2024-11-20T17:49:13,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T17:49:13,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:13,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/23329ca01bce4847bb1831264db671ed is 50, key is test_row_0/B:col10/1732124953540/Put/seqid=0 2024-11-20T17:49:13,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742129_1305 (size=12151) 2024-11-20T17:49:13,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/23329ca01bce4847bb1831264db671ed 2024-11-20T17:49:13,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/06fa114e00344965b644ac05e076aabd is 50, key is test_row_0/C:col10/1732124953540/Put/seqid=0 2024-11-20T17:49:13,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742130_1306 (size=12151) 2024-11-20T17:49:13,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/06fa114e00344965b644ac05e076aabd 2024-11-20T17:49:13,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/62c58e98b9a1488ca6b7055dc865fb13 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/62c58e98b9a1488ca6b7055dc865fb13 2024-11-20T17:49:13,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/62c58e98b9a1488ca6b7055dc865fb13, entries=200, sequenceid=256, filesize=14.2 K 2024-11-20T17:49:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/23329ca01bce4847bb1831264db671ed as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/23329ca01bce4847bb1831264db671ed 2024-11-20T17:49:13,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/23329ca01bce4847bb1831264db671ed, entries=150, sequenceid=256, filesize=11.9 K 2024-11-20T17:49:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/06fa114e00344965b644ac05e076aabd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06fa114e00344965b644ac05e076aabd 2024-11-20T17:49:13,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06fa114e00344965b644ac05e076aabd, entries=150, sequenceid=256, filesize=11.9 K 2024-11-20T17:49:13,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=127.47 KB/130530 for ee91b716d3c9cb4059bcd0deede2ec56 in 158ms, sequenceid=256, compaction requested=true 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:13,701 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:13,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:49:13,701 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61195 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58805 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:13,703 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,703 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,703 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/fe726db31d5145d89510be8f2ddb5190, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d654ff20a29246be8a07c2122a0e1a83, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0a7146134d9e42cdbbeea19708eb753c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/36edd9d8d7384a67a40be0e93dc38e41, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/62c58e98b9a1488ca6b7055dc865fb13] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=59.8 K 2024-11-20T17:49:13,703 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/7db57801e8b64e7cb0bb4ed336879f25, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/91c82e65443d45089e4fbc33173dc0b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2c3f3446bc8d43b3836f33546517789b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6fae00262df140329aa351f6081c66c4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/23329ca01bce4847bb1831264db671ed] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=57.4 K 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db57801e8b64e7cb0bb4ed336879f25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732124949080 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe726db31d5145d89510be8f2ddb5190, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732124949080 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c82e65443d45089e4fbc33173dc0b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124949147 2024-11-20T17:49:13,703 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d654ff20a29246be8a07c2122a0e1a83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124949147 2024-11-20T17:49:13,704 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3f3446bc8d43b3836f33546517789b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732124951311 2024-11-20T17:49:13,704 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a7146134d9e42cdbbeea19708eb753c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732124951311 2024-11-20T17:49:13,704 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fae00262df140329aa351f6081c66c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732124951369 2024-11-20T17:49:13,704 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36edd9d8d7384a67a40be0e93dc38e41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732124951369 2024-11-20T17:49:13,705 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 23329ca01bce4847bb1831264db671ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732124953528 2024-11-20T17:49:13,705 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c58e98b9a1488ca6b7055dc865fb13, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732124952513 2024-11-20T17:49:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:13,707 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:13,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:49:13,717 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:13,718 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/4ae4907e37da49539654e53a85464ec3 is 50, key is test_row_0/A:col10/1732124953540/Put/seqid=0 2024-11-20T17:49:13,718 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:13,719 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3319d6200180480385c638af6f5df3ae is 50, key is test_row_0/B:col10/1732124953540/Put/seqid=0 2024-11-20T17:49:13,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/17de4063cdc84528aaebcef1159b1a58 is 50, key is test_row_0/A:col10/1732124953656/Put/seqid=0 2024-11-20T17:49:13,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742131_1307 (size=12765) 2024-11-20T17:49:13,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742132_1308 (size=12765) 2024-11-20T17:49:13,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T17:49:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T17:49:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125013754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:13,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742133_1309 (size=14741) 2024-11-20T17:49:13,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/17de4063cdc84528aaebcef1159b1a58 2024-11-20T17:49:13,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125013761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:13,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/cb976ddfb3554dc292c1276453bc007b is 50, key is test_row_0/B:col10/1732124953656/Put/seqid=0 2024-11-20T17:49:13,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742134_1310 (size=12301) 2024-11-20T17:49:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125013867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125013877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:13,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T17:49:13,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:13,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:13,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T17:49:14,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:14,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:14,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:14,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T17:49:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125014077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125014087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,143 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/4ae4907e37da49539654e53a85464ec3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/4ae4907e37da49539654e53a85464ec3 2024-11-20T17:49:14,148 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into 4ae4907e37da49539654e53a85464ec3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:14,148 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:14,148 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=11, startTime=1732124953701; duration=0sec 2024-11-20T17:49:14,148 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:14,149 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:14,149 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T17:49:14,150 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58805 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T17:49:14,150 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:14,150 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:14,150 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/549f140ffa974633b6b5f76417ab392e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/08ec618d6ff54cbea90c47e21f148e49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/412ee5258fc14ba99ffafa643d2e6647, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/44bf9b52003744db8626a26ef65e8c1a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06fa114e00344965b644ac05e076aabd] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=57.4 K 2024-11-20T17:49:14,151 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549f140ffa974633b6b5f76417ab392e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732124949080 2024-11-20T17:49:14,151 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ec618d6ff54cbea90c47e21f148e49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732124949147 2024-11-20T17:49:14,152 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3319d6200180480385c638af6f5df3ae as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3319d6200180480385c638af6f5df3ae 2024-11-20T17:49:14,152 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412ee5258fc14ba99ffafa643d2e6647, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732124951311 2024-11-20T17:49:14,152 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44bf9b52003744db8626a26ef65e8c1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732124951369 2024-11-20T17:49:14,153 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06fa114e00344965b644ac05e076aabd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732124953528 2024-11-20T17:49:14,157 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 3319d6200180480385c638af6f5df3ae(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:14,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:14,158 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=11, startTime=1732124953701; duration=0sec 2024-11-20T17:49:14,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:14,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:14,171 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#261 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:14,171 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/0071b39db9724b1f84c5d17f8d3841cd is 50, key is test_row_0/C:col10/1732124953540/Put/seqid=0 2024-11-20T17:49:14,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742135_1311 (size=12765) 2024-11-20T17:49:14,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/cb976ddfb3554dc292c1276453bc007b 2024-11-20T17:49:14,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T17:49:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:14,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b6de0457fe3143d6a5afec1f8efa4d2c is 50, key is test_row_0/C:col10/1732124953656/Put/seqid=0 2024-11-20T17:49:14,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742136_1312 (size=12301) 2024-11-20T17:49:14,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b6de0457fe3143d6a5afec1f8efa4d2c 2024-11-20T17:49:14,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/17de4063cdc84528aaebcef1159b1a58 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/17de4063cdc84528aaebcef1159b1a58 2024-11-20T17:49:14,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/17de4063cdc84528aaebcef1159b1a58, entries=200, sequenceid=280, filesize=14.4 K 2024-11-20T17:49:14,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/cb976ddfb3554dc292c1276453bc007b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/cb976ddfb3554dc292c1276453bc007b 2024-11-20T17:49:14,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/cb976ddfb3554dc292c1276453bc007b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-20T17:49:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b6de0457fe3143d6a5afec1f8efa4d2c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b6de0457fe3143d6a5afec1f8efa4d2c 2024-11-20T17:49:14,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b6de0457fe3143d6a5afec1f8efa4d2c, entries=150, sequenceid=280, filesize=12.0 K 2024-11-20T17:49:14,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ee91b716d3c9cb4059bcd0deede2ec56 in 530ms, sequenceid=280, compaction requested=false 2024-11-20T17:49:14,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:14,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T17:49:14,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:14,365 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:49:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/91dfc5f1d493446a8ea00684e2c385cc is 50, key is test_row_0/A:col10/1732124953759/Put/seqid=0 2024-11-20T17:49:14,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742137_1313 (size=12301) 2024-11-20T17:49:14,374 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/91dfc5f1d493446a8ea00684e2c385cc 2024-11-20T17:49:14,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/8809f20752ca461b85a7ae847224d347 is 50, key is test_row_0/B:col10/1732124953759/Put/seqid=0 2024-11-20T17:49:14,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742138_1314 (size=12301) 2024-11-20T17:49:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:14,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125014466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125014467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T17:49:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125014577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125014577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,587 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/0071b39db9724b1f84c5d17f8d3841cd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/0071b39db9724b1f84c5d17f8d3841cd 2024-11-20T17:49:14,592 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into 0071b39db9724b1f84c5d17f8d3841cd(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:14,592 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:14,592 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=11, startTime=1732124953701; duration=0sec 2024-11-20T17:49:14,592 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:14,592 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:14,784 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/8809f20752ca461b85a7ae847224d347 2024-11-20T17:49:14,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125014787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125014787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:14,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/398f54a7aa684674a7d87a4a5dc5676a is 50, key is test_row_0/C:col10/1732124953759/Put/seqid=0 2024-11-20T17:49:14,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742139_1315 (size=12301) 2024-11-20T17:49:15,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125015097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125015097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,200 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/398f54a7aa684674a7d87a4a5dc5676a 2024-11-20T17:49:15,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/91dfc5f1d493446a8ea00684e2c385cc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/91dfc5f1d493446a8ea00684e2c385cc 2024-11-20T17:49:15,215 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/91dfc5f1d493446a8ea00684e2c385cc, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:49:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/8809f20752ca461b85a7ae847224d347 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/8809f20752ca461b85a7ae847224d347 2024-11-20T17:49:15,221 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/8809f20752ca461b85a7ae847224d347, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:49:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/398f54a7aa684674a7d87a4a5dc5676a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/398f54a7aa684674a7d87a4a5dc5676a 2024-11-20T17:49:15,230 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/398f54a7aa684674a7d87a4a5dc5676a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:49:15,232 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ee91b716d3c9cb4059bcd0deede2ec56 in 867ms, sequenceid=294, compaction requested=true 2024-11-20T17:49:15,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:15,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:15,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-20T17:49:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-20T17:49:15,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T17:49:15,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7930 sec 2024-11-20T17:49:15,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7970 sec 2024-11-20T17:49:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T17:49:15,587 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:15,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-20T17:49:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T17:49:15,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:15,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:15,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:15,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:15,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:15,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/31500503edff46dea00c9a7e1eb07d19 is 50, key is test_row_0/A:col10/1732124955609/Put/seqid=0 2024-11-20T17:49:15,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742140_1316 (size=14741) 2024-11-20T17:49:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125015624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125015628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T17:49:15,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T17:49:15,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125015738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:15,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125015738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:15,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T17:49:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:15,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T17:49:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125015947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125015947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/31500503edff46dea00c9a7e1eb07d19 2024-11-20T17:49:16,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/b1739eb283524115a7961aa2b9c392e2 is 50, key is test_row_0/B:col10/1732124955609/Put/seqid=0 2024-11-20T17:49:16,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742141_1317 (size=12301) 2024-11-20T17:49:16,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/b1739eb283524115a7961aa2b9c392e2 2024-11-20T17:49:16,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/9d4a0741dbd74a65b810ea4192ee40d2 is 50, key is test_row_0/C:col10/1732124955609/Put/seqid=0 2024-11-20T17:49:16,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742142_1318 (size=12301) 2024-11-20T17:49:16,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T17:49:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T17:49:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T17:49:16,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125016257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:16,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125016257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T17:49:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:16,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/9d4a0741dbd74a65b810ea4192ee40d2 2024-11-20T17:49:16,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/31500503edff46dea00c9a7e1eb07d19 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/31500503edff46dea00c9a7e1eb07d19 2024-11-20T17:49:16,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/31500503edff46dea00c9a7e1eb07d19, entries=200, sequenceid=321, filesize=14.4 K 2024-11-20T17:49:16,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/b1739eb283524115a7961aa2b9c392e2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/b1739eb283524115a7961aa2b9c392e2 2024-11-20T17:49:16,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/b1739eb283524115a7961aa2b9c392e2, entries=150, sequenceid=321, filesize=12.0 K 2024-11-20T17:49:16,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/9d4a0741dbd74a65b810ea4192ee40d2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9d4a0741dbd74a65b810ea4192ee40d2 2024-11-20T17:49:16,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9d4a0741dbd74a65b810ea4192ee40d2, entries=150, sequenceid=321, filesize=12.0 K 2024-11-20T17:49:16,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ee91b716d3c9cb4059bcd0deede2ec56 in 854ms, sequenceid=321, compaction requested=true 2024-11-20T17:49:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:16,464 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:16,464 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:16,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:16,465 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:16,465 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:16,465 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:16,465 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:16,466 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,466 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,466 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3319d6200180480385c638af6f5df3ae, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/cb976ddfb3554dc292c1276453bc007b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/8809f20752ca461b85a7ae847224d347, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/b1739eb283524115a7961aa2b9c392e2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=48.5 K 2024-11-20T17:49:16,466 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/4ae4907e37da49539654e53a85464ec3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/17de4063cdc84528aaebcef1159b1a58, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/91dfc5f1d493446a8ea00684e2c385cc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/31500503edff46dea00c9a7e1eb07d19] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=53.3 K 2024-11-20T17:49:16,466 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3319d6200180480385c638af6f5df3ae, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732124953528 2024-11-20T17:49:16,466 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae4907e37da49539654e53a85464ec3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732124953528 2024-11-20T17:49:16,466 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting cb976ddfb3554dc292c1276453bc007b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732124953656 2024-11-20T17:49:16,467 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17de4063cdc84528aaebcef1159b1a58, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732124953656 2024-11-20T17:49:16,467 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8809f20752ca461b85a7ae847224d347, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124953742 2024-11-20T17:49:16,467 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91dfc5f1d493446a8ea00684e2c385cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124953742 2024-11-20T17:49:16,467 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b1739eb283524115a7961aa2b9c392e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732124954464 2024-11-20T17:49:16,467 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31500503edff46dea00c9a7e1eb07d19, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732124954464 2024-11-20T17:49:16,475 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:16,476 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/a0b39af7e81943939204409d9a2c40e3 is 50, key is test_row_0/B:col10/1732124955609/Put/seqid=0 2024-11-20T17:49:16,476 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:16,477 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/cc4ee0d99f054e7c8058edc2667d033f is 50, key is test_row_0/A:col10/1732124955609/Put/seqid=0 2024-11-20T17:49:16,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742143_1319 (size=13051) 2024-11-20T17:49:16,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742144_1320 (size=13051) 2024-11-20T17:49:16,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T17:49:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,506 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T17:49:16,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/1383d6620e5c4e749f75da5f86b854d3 is 50, key is test_row_0/A:col10/1732124955622/Put/seqid=0 2024-11-20T17:49:16,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742145_1321 (size=12301) 2024-11-20T17:49:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T17:49:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:16,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125016813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125016818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,892 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/cc4ee0d99f054e7c8058edc2667d033f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/cc4ee0d99f054e7c8058edc2667d033f 2024-11-20T17:49:16,892 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/a0b39af7e81943939204409d9a2c40e3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a0b39af7e81943939204409d9a2c40e3 2024-11-20T17:49:16,898 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into a0b39af7e81943939204409d9a2c40e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:16,898 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:16,898 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=12, startTime=1732124956464; duration=0sec 2024-11-20T17:49:16,898 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:16,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:16,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:16,899 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into cc4ee0d99f054e7c8058edc2667d033f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:16,899 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:16,899 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=12, startTime=1732124956464; duration=0sec 2024-11-20T17:49:16,899 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:16,899 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:16,900 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:16,900 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:16,900 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,900 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/0071b39db9724b1f84c5d17f8d3841cd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b6de0457fe3143d6a5afec1f8efa4d2c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/398f54a7aa684674a7d87a4a5dc5676a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9d4a0741dbd74a65b810ea4192ee40d2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=48.5 K 2024-11-20T17:49:16,901 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0071b39db9724b1f84c5d17f8d3841cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732124953528 2024-11-20T17:49:16,901 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b6de0457fe3143d6a5afec1f8efa4d2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732124953656 2024-11-20T17:49:16,901 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 398f54a7aa684674a7d87a4a5dc5676a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124953742 2024-11-20T17:49:16,902 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4a0741dbd74a65b810ea4192ee40d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732124954464 2024-11-20T17:49:16,910 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:16,911 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/06328f5496804a9993ca9f573389ea85 is 50, key is test_row_0/C:col10/1732124955609/Put/seqid=0 2024-11-20T17:49:16,917 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/1383d6620e5c4e749f75da5f86b854d3 2024-11-20T17:49:16,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/efcdd6c21f514de4b9baf863aecbc000 is 50, key is test_row_0/B:col10/1732124955622/Put/seqid=0 2024-11-20T17:49:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125016927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125016928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:16,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742146_1322 (size=13051) 2024-11-20T17:49:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:16,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742147_1323 (size=12301) 2024-11-20T17:49:16,933 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/efcdd6c21f514de4b9baf863aecbc000 2024-11-20T17:49:16,937 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/06328f5496804a9993ca9f573389ea85 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06328f5496804a9993ca9f573389ea85 2024-11-20T17:49:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/50ba5a408ca445cf89678d94fbe534a0 is 50, key is test_row_0/C:col10/1732124955622/Put/seqid=0 2024-11-20T17:49:16,943 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into 06328f5496804a9993ca9f573389ea85(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:16,943 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:16,943 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=12, startTime=1732124956464; duration=0sec 2024-11-20T17:49:16,944 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:16,944 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:16,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742148_1324 (size=12301) 2024-11-20T17:49:16,950 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/50ba5a408ca445cf89678d94fbe534a0 2024-11-20T17:49:16,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/1383d6620e5c4e749f75da5f86b854d3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1383d6620e5c4e749f75da5f86b854d3 2024-11-20T17:49:16,962 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1383d6620e5c4e749f75da5f86b854d3, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T17:49:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/efcdd6c21f514de4b9baf863aecbc000 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/efcdd6c21f514de4b9baf863aecbc000 2024-11-20T17:49:16,968 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/efcdd6c21f514de4b9baf863aecbc000, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T17:49:16,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/50ba5a408ca445cf89678d94fbe534a0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/50ba5a408ca445cf89678d94fbe534a0 2024-11-20T17:49:16,975 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/50ba5a408ca445cf89678d94fbe534a0, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T17:49:16,976 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ee91b716d3c9cb4059bcd0deede2ec56 in 470ms, sequenceid=331, compaction requested=false 2024-11-20T17:49:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-20T17:49:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-20T17:49:16,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T17:49:16,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3880 sec 2024-11-20T17:49:16,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.3920 sec 2024-11-20T17:49:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:17,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T17:49:17,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/815dd3f509ba4ed296960923f6a3061a is 50, key is test_row_0/A:col10/1732124957139/Put/seqid=0 2024-11-20T17:49:17,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742149_1325 (size=12301) 2024-11-20T17:49:17,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/815dd3f509ba4ed296960923f6a3061a 2024-11-20T17:49:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125017151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125017152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/9c2109ca4cef4d6997ff7f7ae83d6e4e is 50, key is test_row_0/B:col10/1732124957139/Put/seqid=0 2024-11-20T17:49:17,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742150_1326 (size=12301) 2024-11-20T17:49:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125017257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125017267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125017467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125017477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/9c2109ca4cef4d6997ff7f7ae83d6e4e 2024-11-20T17:49:17,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/fc4ad5410ddd493f9c8f3d935ad3cf31 is 50, key is test_row_0/C:col10/1732124957139/Put/seqid=0 2024-11-20T17:49:17,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742151_1327 (size=12301) 2024-11-20T17:49:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T17:49:17,738 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:17,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-20T17:49:17,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T17:49:17,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:17,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:17,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125017777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125017787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T17:49:17,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T17:49:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:17,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:17,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/fc4ad5410ddd493f9c8f3d935ad3cf31 2024-11-20T17:49:17,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/815dd3f509ba4ed296960923f6a3061a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/815dd3f509ba4ed296960923f6a3061a 2024-11-20T17:49:17,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/815dd3f509ba4ed296960923f6a3061a, entries=150, sequenceid=362, filesize=12.0 K 2024-11-20T17:49:17,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/9c2109ca4cef4d6997ff7f7ae83d6e4e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9c2109ca4cef4d6997ff7f7ae83d6e4e 2024-11-20T17:49:18,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9c2109ca4cef4d6997ff7f7ae83d6e4e, entries=150, sequenceid=362, filesize=12.0 K 2024-11-20T17:49:18,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/fc4ad5410ddd493f9c8f3d935ad3cf31 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fc4ad5410ddd493f9c8f3d935ad3cf31 2024-11-20T17:49:18,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fc4ad5410ddd493f9c8f3d935ad3cf31, entries=150, sequenceid=362, filesize=12.0 K 2024-11-20T17:49:18,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for ee91b716d3c9cb4059bcd0deede2ec56 in 867ms, sequenceid=362, compaction requested=true 2024-11-20T17:49:18,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:18,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:18,008 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:18,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:18,008 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:18,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:18,009 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:18,009 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:18,009 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,009 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/cc4ee0d99f054e7c8058edc2667d033f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1383d6620e5c4e749f75da5f86b854d3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/815dd3f509ba4ed296960923f6a3061a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=36.8 K 2024-11-20T17:49:18,010 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:18,010 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:18,010 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,010 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a0b39af7e81943939204409d9a2c40e3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/efcdd6c21f514de4b9baf863aecbc000, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9c2109ca4cef4d6997ff7f7ae83d6e4e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=36.8 K 2024-11-20T17:49:18,010 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc4ee0d99f054e7c8058edc2667d033f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732124954464 2024-11-20T17:49:18,010 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b39af7e81943939204409d9a2c40e3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732124954464 2024-11-20T17:49:18,011 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1383d6620e5c4e749f75da5f86b854d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732124955622 2024-11-20T17:49:18,011 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting efcdd6c21f514de4b9baf863aecbc000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732124955622 2024-11-20T17:49:18,011 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 815dd3f509ba4ed296960923f6a3061a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732124956806 2024-11-20T17:49:18,011 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2109ca4cef4d6997ff7f7ae83d6e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732124956806 2024-11-20T17:49:18,018 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:18,018 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/7c6534ec40fd4dd6b54ca4ad9a89067e is 50, key is test_row_0/A:col10/1732124957139/Put/seqid=0 2024-11-20T17:49:18,026 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:18,027 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/aefbb8bad637473a9f54d5e81af3c4b8 is 50, key is test_row_0/B:col10/1732124957139/Put/seqid=0 2024-11-20T17:49:18,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742152_1328 (size=13153) 2024-11-20T17:49:18,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T17:49:18,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,046 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T17:49:18,047 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/7c6534ec40fd4dd6b54ca4ad9a89067e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/7c6534ec40fd4dd6b54ca4ad9a89067e 2024-11-20T17:49:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/24662350dac041cd8cb9626fb8d82710 is 50, key is test_row_0/A:col10/1732124957145/Put/seqid=0 2024-11-20T17:49:18,055 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into 7c6534ec40fd4dd6b54ca4ad9a89067e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:18,055 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:18,055 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=13, startTime=1732124958008; duration=0sec 2024-11-20T17:49:18,055 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:18,055 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:18,056 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:18,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742153_1329 (size=13153) 2024-11-20T17:49:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T17:49:18,058 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:18,058 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:18,058 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,058 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06328f5496804a9993ca9f573389ea85, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/50ba5a408ca445cf89678d94fbe534a0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fc4ad5410ddd493f9c8f3d935ad3cf31] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=36.8 K 2024-11-20T17:49:18,061 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06328f5496804a9993ca9f573389ea85, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732124954464 2024-11-20T17:49:18,062 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ba5a408ca445cf89678d94fbe534a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732124955622 2024-11-20T17:49:18,063 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc4ad5410ddd493f9c8f3d935ad3cf31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732124956806 2024-11-20T17:49:18,064 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/aefbb8bad637473a9f54d5e81af3c4b8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/aefbb8bad637473a9f54d5e81af3c4b8 2024-11-20T17:49:18,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742154_1330 (size=12301) 2024-11-20T17:49:18,068 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/24662350dac041cd8cb9626fb8d82710 2024-11-20T17:49:18,070 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into aefbb8bad637473a9f54d5e81af3c4b8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:18,070 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:18,070 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=13, startTime=1732124958008; duration=0sec 2024-11-20T17:49:18,070 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:18,070 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:18,074 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:18,075 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b34b50f272ac48cead2460e0aec75cb4 is 50, key is test_row_0/C:col10/1732124957139/Put/seqid=0 2024-11-20T17:49:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/f3126f3965ab41199203a0d28d0271d7 is 50, key is test_row_0/B:col10/1732124957145/Put/seqid=0 2024-11-20T17:49:18,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742155_1331 (size=13153) 2024-11-20T17:49:18,106 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b34b50f272ac48cead2460e0aec75cb4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b34b50f272ac48cead2460e0aec75cb4 2024-11-20T17:49:18,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742156_1332 (size=12301) 2024-11-20T17:49:18,106 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/f3126f3965ab41199203a0d28d0271d7 2024-11-20T17:49:18,110 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into b34b50f272ac48cead2460e0aec75cb4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:18,111 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:18,111 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=13, startTime=1732124958009; duration=0sec 2024-11-20T17:49:18,111 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:18,111 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:18,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/19f2d4cc9aca485f91d764b4b9e91461 is 50, key is test_row_0/C:col10/1732124957145/Put/seqid=0 2024-11-20T17:49:18,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742157_1333 (size=12301) 2024-11-20T17:49:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:18,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125018335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125018339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T17:49:18,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125018447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125018447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,524 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/19f2d4cc9aca485f91d764b4b9e91461 2024-11-20T17:49:18,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/24662350dac041cd8cb9626fb8d82710 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/24662350dac041cd8cb9626fb8d82710 2024-11-20T17:49:18,532 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/24662350dac041cd8cb9626fb8d82710, entries=150, sequenceid=370, filesize=12.0 K 2024-11-20T17:49:18,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/f3126f3965ab41199203a0d28d0271d7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/f3126f3965ab41199203a0d28d0271d7 2024-11-20T17:49:18,536 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/f3126f3965ab41199203a0d28d0271d7, entries=150, sequenceid=370, filesize=12.0 K 2024-11-20T17:49:18,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/19f2d4cc9aca485f91d764b4b9e91461 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19f2d4cc9aca485f91d764b4b9e91461 2024-11-20T17:49:18,543 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19f2d4cc9aca485f91d764b4b9e91461, entries=150, sequenceid=370, filesize=12.0 K 2024-11-20T17:49:18,544 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for ee91b716d3c9cb4059bcd0deede2ec56 in 498ms, sequenceid=370, compaction requested=false 2024-11-20T17:49:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-20T17:49:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-20T17:49:18,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-20T17:49:18,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 806 msec 2024-11-20T17:49:18,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 810 msec 2024-11-20T17:49:18,675 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:18,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-20T17:49:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:18,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5ba9879965b64223a30a0c7a528c5347 is 50, key is test_row_0/A:col10/1732124958337/Put/seqid=0 2024-11-20T17:49:18,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125018689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125018696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742158_1334 (size=12301) 2024-11-20T17:49:18,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5ba9879965b64223a30a0c7a528c5347 2024-11-20T17:49:18,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/95ae73502e23483495a0b80f26d08391 is 50, key is test_row_0/B:col10/1732124958337/Put/seqid=0 2024-11-20T17:49:18,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742159_1335 (size=12301) 2024-11-20T17:49:18,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/95ae73502e23483495a0b80f26d08391 2024-11-20T17:49:18,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b4a939a8a65044e7bf1c7193c993cb7a is 50, key is test_row_0/C:col10/1732124958337/Put/seqid=0 2024-11-20T17:49:18,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742160_1336 (size=12301) 2024-11-20T17:49:18,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b4a939a8a65044e7bf1c7193c993cb7a 2024-11-20T17:49:18,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5ba9879965b64223a30a0c7a528c5347 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5ba9879965b64223a30a0c7a528c5347 2024-11-20T17:49:18,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5ba9879965b64223a30a0c7a528c5347, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T17:49:18,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/95ae73502e23483495a0b80f26d08391 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/95ae73502e23483495a0b80f26d08391 2024-11-20T17:49:18,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/95ae73502e23483495a0b80f26d08391, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T17:49:18,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/b4a939a8a65044e7bf1c7193c993cb7a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b4a939a8a65044e7bf1c7193c993cb7a 2024-11-20T17:49:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125018807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125018808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:18,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b4a939a8a65044e7bf1c7193c993cb7a, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T17:49:18,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for ee91b716d3c9cb4059bcd0deede2ec56 in 137ms, sequenceid=403, compaction requested=true 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:18,813 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:18,813 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:18,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:18,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,814 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:18,814 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/7c6534ec40fd4dd6b54ca4ad9a89067e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/24662350dac041cd8cb9626fb8d82710, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5ba9879965b64223a30a0c7a528c5347] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=36.9 K 2024-11-20T17:49:18,814 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/aefbb8bad637473a9f54d5e81af3c4b8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/f3126f3965ab41199203a0d28d0271d7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/95ae73502e23483495a0b80f26d08391] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=36.9 K 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6534ec40fd4dd6b54ca4ad9a89067e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732124956806 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting aefbb8bad637473a9f54d5e81af3c4b8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732124956806 2024-11-20T17:49:18,814 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24662350dac041cd8cb9626fb8d82710, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732124957145 2024-11-20T17:49:18,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f3126f3965ab41199203a0d28d0271d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732124957145 2024-11-20T17:49:18,815 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba9879965b64223a30a0c7a528c5347, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124958334 2024-11-20T17:49:18,815 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ae73502e23483495a0b80f26d08391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124958334 2024-11-20T17:49:18,834 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:18,835 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f8d64b38c7ce4cb9991bce4b72528c05 is 50, key is test_row_0/A:col10/1732124958337/Put/seqid=0 2024-11-20T17:49:18,842 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:18,843 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/938f80bd70fe43e9a2dd04e31fffd950 is 50, key is test_row_0/B:col10/1732124958337/Put/seqid=0 2024-11-20T17:49:18,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742161_1337 (size=13255) 2024-11-20T17:49:18,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742162_1338 (size=13255) 2024-11-20T17:49:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T17:49:18,877 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:18,877 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-20T17:49:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T17:49:18,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:18,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:18,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T17:49:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:19,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:19,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d739a29c491f428fb544dcae7d8ba8c5 is 50, key is test_row_0/A:col10/1732124959031/Put/seqid=0 2024-11-20T17:49:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742163_1339 (size=14741) 2024-11-20T17:49:19,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125019083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125019085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T17:49:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125019197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125019197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,265 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f8d64b38c7ce4cb9991bce4b72528c05 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f8d64b38c7ce4cb9991bce4b72528c05 2024-11-20T17:49:19,274 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into f8d64b38c7ce4cb9991bce4b72528c05(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:19,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:19,274 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=13, startTime=1732124958813; duration=0sec 2024-11-20T17:49:19,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:19,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:19,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:19,275 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/938f80bd70fe43e9a2dd04e31fffd950 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/938f80bd70fe43e9a2dd04e31fffd950 2024-11-20T17:49:19,275 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:19,275 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:19,275 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,276 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b34b50f272ac48cead2460e0aec75cb4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19f2d4cc9aca485f91d764b4b9e91461, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b4a939a8a65044e7bf1c7193c993cb7a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=36.9 K 2024-11-20T17:49:19,276 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b34b50f272ac48cead2460e0aec75cb4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732124956806 2024-11-20T17:49:19,277 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f2d4cc9aca485f91d764b4b9e91461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732124957145 2024-11-20T17:49:19,278 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a939a8a65044e7bf1c7193c993cb7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124958334 2024-11-20T17:49:19,283 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 938f80bd70fe43e9a2dd04e31fffd950(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:19,283 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:19,283 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=13, startTime=1732124958813; duration=0sec 2024-11-20T17:49:19,283 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:19,283 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:19,285 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:19,286 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/19698cc527eb4b5c8752e8583f35b66d is 50, key is test_row_0/C:col10/1732124958337/Put/seqid=0 2024-11-20T17:49:19,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742164_1340 (size=13255) 2024-11-20T17:49:19,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125019407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125019407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d739a29c491f428fb544dcae7d8ba8c5 2024-11-20T17:49:19,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/33e8b1d3c20c4128893503a89cb101d9 is 50, key is test_row_0/B:col10/1732124959031/Put/seqid=0 2024-11-20T17:49:19,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742165_1341 (size=12301) 2024-11-20T17:49:19,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T17:49:19,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,695 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/19698cc527eb4b5c8752e8583f35b66d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19698cc527eb4b5c8752e8583f35b66d 2024-11-20T17:49:19,699 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into 19698cc527eb4b5c8752e8583f35b66d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:19,699 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:19,699 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=13, startTime=1732124958813; duration=0sec 2024-11-20T17:49:19,699 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:19,699 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125019718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125019718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:19,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/33e8b1d3c20c4128893503a89cb101d9 2024-11-20T17:49:19,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/565ae948fb4e46e5a796dc66c68b9943 is 50, key is test_row_0/C:col10/1732124959031/Put/seqid=0 2024-11-20T17:49:19,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742166_1342 (size=12301) 2024-11-20T17:49:19,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:19,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T17:49:20,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:20,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125020227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:20,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125020227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:20,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:20,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:20,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:20,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:20,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:20,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/565ae948fb4e46e5a796dc66c68b9943 2024-11-20T17:49:20,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/d739a29c491f428fb544dcae7d8ba8c5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d739a29c491f428fb544dcae7d8ba8c5 2024-11-20T17:49:20,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d739a29c491f428fb544dcae7d8ba8c5, entries=200, sequenceid=415, filesize=14.4 K 2024-11-20T17:49:20,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/33e8b1d3c20c4128893503a89cb101d9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/33e8b1d3c20c4128893503a89cb101d9 2024-11-20T17:49:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/33e8b1d3c20c4128893503a89cb101d9, entries=150, sequenceid=415, filesize=12.0 K 2024-11-20T17:49:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/565ae948fb4e46e5a796dc66c68b9943 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/565ae948fb4e46e5a796dc66c68b9943 2024-11-20T17:49:20,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/565ae948fb4e46e5a796dc66c68b9943, entries=150, sequenceid=415, filesize=12.0 K 2024-11-20T17:49:20,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ee91b716d3c9cb4059bcd0deede2ec56 in 1261ms, sequenceid=415, compaction requested=false 2024-11-20T17:49:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:20,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T17:49:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:20,415 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:49:20,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f23c3894a36e4fb78811177b51a67116 is 50, key is test_row_0/A:col10/1732124959081/Put/seqid=0 2024-11-20T17:49:20,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742167_1343 (size=12301) 2024-11-20T17:49:20,435 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f23c3894a36e4fb78811177b51a67116 2024-11-20T17:49:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3c060daec37e40bd8b87ff6d95cb94c0 is 50, key is test_row_0/B:col10/1732124959081/Put/seqid=0 2024-11-20T17:49:20,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742168_1344 (size=12301) 2024-11-20T17:49:20,482 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3c060daec37e40bd8b87ff6d95cb94c0 2024-11-20T17:49:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/a281b586532946a4b3ca9d3e588a9c46 is 50, key is test_row_0/C:col10/1732124959081/Put/seqid=0 2024-11-20T17:49:20,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742169_1345 (size=12301) 2024-11-20T17:49:20,898 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/a281b586532946a4b3ca9d3e588a9c46 2024-11-20T17:49:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/f23c3894a36e4fb78811177b51a67116 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f23c3894a36e4fb78811177b51a67116 2024-11-20T17:49:20,905 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f23c3894a36e4fb78811177b51a67116, entries=150, sequenceid=442, filesize=12.0 K 2024-11-20T17:49:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3c060daec37e40bd8b87ff6d95cb94c0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3c060daec37e40bd8b87ff6d95cb94c0 2024-11-20T17:49:20,910 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3c060daec37e40bd8b87ff6d95cb94c0, entries=150, sequenceid=442, filesize=12.0 K 2024-11-20T17:49:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/a281b586532946a4b3ca9d3e588a9c46 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/a281b586532946a4b3ca9d3e588a9c46 2024-11-20T17:49:20,916 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/a281b586532946a4b3ca9d3e588a9c46, entries=150, sequenceid=442, filesize=12.0 K 2024-11-20T17:49:20,916 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for ee91b716d3c9cb4059bcd0deede2ec56 in 501ms, sequenceid=442, compaction requested=true 2024-11-20T17:49:20,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:20,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:20,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-20T17:49:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-20T17:49:20,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T17:49:20,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-20T17:49:20,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.0430 sec 2024-11-20T17:49:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T17:49:21,027 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:21,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-20T17:49:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T17:49:21,030 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:21,030 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:21,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T17:49:21,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:49:21,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-20T17:49:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-20T17:49:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-20T17:49:21,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T17:49:21,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 153 msec 2024-11-20T17:49:21,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 158 msec 2024-11-20T17:49:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:21,248 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:21,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:21,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/902f88ee1a974d42b4f8fa6666ee8aed is 50, key is test_row_0/A:col10/1732124961247/Put/seqid=0 2024-11-20T17:49:21,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742170_1346 (size=12297) 2024-11-20T17:49:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125021292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125021295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T17:49:21,348 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:21,348 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-20T17:49:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:21,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:21,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:21,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125021397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125021407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:21,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T17:49:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125021607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125021617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T17:49:21,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:21,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:21,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/902f88ee1a974d42b4f8fa6666ee8aed 2024-11-20T17:49:21,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/0ef95fe888cd4f9cb5ae859147f14b30 is 50, key is test_row_0/B:col10/1732124961247/Put/seqid=0 2024-11-20T17:49:21,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742171_1347 (size=9857) 2024-11-20T17:49:21,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T17:49:21,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:21,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125021917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125021927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:21,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:21,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T17:49:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:21,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:22,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/0ef95fe888cd4f9cb5ae859147f14b30 2024-11-20T17:49:22,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/060f5fbc42414c4dae541f80b9b9c156 is 50, key is test_row_0/C:col10/1732124961247/Put/seqid=0 2024-11-20T17:49:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742172_1348 (size=9857) 2024-11-20T17:49:22,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/060f5fbc42414c4dae541f80b9b9c156 2024-11-20T17:49:22,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/902f88ee1a974d42b4f8fa6666ee8aed as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/902f88ee1a974d42b4f8fa6666ee8aed 2024-11-20T17:49:22,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/902f88ee1a974d42b4f8fa6666ee8aed, entries=150, sequenceid=454, filesize=12.0 K 2024-11-20T17:49:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/0ef95fe888cd4f9cb5ae859147f14b30 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/0ef95fe888cd4f9cb5ae859147f14b30 2024-11-20T17:49:22,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T17:49:22,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/0ef95fe888cd4f9cb5ae859147f14b30, entries=100, sequenceid=454, filesize=9.6 K 2024-11-20T17:49:22,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:22,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:22,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:22,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:22,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/060f5fbc42414c4dae541f80b9b9c156 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/060f5fbc42414c4dae541f80b9b9c156 2024-11-20T17:49:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:22,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/060f5fbc42414c4dae541f80b9b9c156, entries=100, sequenceid=454, filesize=9.6 K 2024-11-20T17:49:22,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ee91b716d3c9cb4059bcd0deede2ec56 in 872ms, sequenceid=454, compaction requested=true 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:22,120 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:22,120 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:22,122 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:22,122 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:22,122 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:22,122 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/938f80bd70fe43e9a2dd04e31fffd950, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/33e8b1d3c20c4128893503a89cb101d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3c060daec37e40bd8b87ff6d95cb94c0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/0ef95fe888cd4f9cb5ae859147f14b30] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=46.6 K 2024-11-20T17:49:22,122 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:22,122 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 938f80bd70fe43e9a2dd04e31fffd950, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124958334 2024-11-20T17:49:22,122 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:22,122 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:22,123 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f8d64b38c7ce4cb9991bce4b72528c05, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d739a29c491f428fb544dcae7d8ba8c5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f23c3894a36e4fb78811177b51a67116, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/902f88ee1a974d42b4f8fa6666ee8aed] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=51.4 K 2024-11-20T17:49:22,123 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e8b1d3c20c4128893503a89cb101d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732124959023 2024-11-20T17:49:22,123 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d64b38c7ce4cb9991bce4b72528c05, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124958334 2024-11-20T17:49:22,123 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c060daec37e40bd8b87ff6d95cb94c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732124959075 2024-11-20T17:49:22,123 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d739a29c491f428fb544dcae7d8ba8c5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732124959022 2024-11-20T17:49:22,123 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef95fe888cd4f9cb5ae859147f14b30, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732124961245 2024-11-20T17:49:22,124 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f23c3894a36e4fb78811177b51a67116, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732124959075 2024-11-20T17:49:22,124 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 902f88ee1a974d42b4f8fa6666ee8aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732124961245 2024-11-20T17:49:22,132 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:22,132 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:22,132 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/9a73604d34914040a25cbef052beacd1 is 50, key is test_row_0/A:col10/1732124961247/Put/seqid=0 2024-11-20T17:49:22,132 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/55a74b69bbfb418fb8c739c87d9b143b is 50, key is test_row_0/B:col10/1732124961247/Put/seqid=0 2024-11-20T17:49:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742173_1349 (size=13391) 2024-11-20T17:49:22,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742174_1350 (size=13391) 2024-11-20T17:49:22,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T17:49:22,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:22,267 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:22,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5f717f779b7944478846cec7da6e6503 is 50, key is test_row_0/A:col10/1732124961293/Put/seqid=0 2024-11-20T17:49:22,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742175_1351 (size=12301) 2024-11-20T17:49:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:22,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. as already flushing 2024-11-20T17:49:22,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125022447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125022451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:22,549 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-11-20T17:49:22,549 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:22,549 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,550 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-20T17:49:22,550 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:22,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:22,550 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,551 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/9a73604d34914040a25cbef052beacd1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/9a73604d34914040a25cbef052beacd1 2024-11-20T17:49:22,551 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,552 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:22,553 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-20T17:49:22,553 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:22,553 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,553 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:22,554 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-11-20T17:49:22,554 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:22,554 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,554 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,554 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/55a74b69bbfb418fb8c739c87d9b143b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/55a74b69bbfb418fb8c739c87d9b143b 2024-11-20T17:49:22,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:22,554 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-11-20T17:49:22,554 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:22,554 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,555 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:22,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:22,555 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into 9a73604d34914040a25cbef052beacd1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:22,555 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:22,555 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=12, startTime=1732124962120; duration=0sec 2024-11-20T17:49:22,555 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:22,556 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:22,556 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:22,556 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:22,557 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:22,557 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:22,557 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19698cc527eb4b5c8752e8583f35b66d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/565ae948fb4e46e5a796dc66c68b9943, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/a281b586532946a4b3ca9d3e588a9c46, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/060f5fbc42414c4dae541f80b9b9c156] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=46.6 K 2024-11-20T17:49:22,557 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19698cc527eb4b5c8752e8583f35b66d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732124958334 2024-11-20T17:49:22,557 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 565ae948fb4e46e5a796dc66c68b9943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732124959023 2024-11-20T17:49:22,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125022557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,557 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a281b586532946a4b3ca9d3e588a9c46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732124959075 2024-11-20T17:49:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,558 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 060f5fbc42414c4dae541f80b9b9c156, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732124961245 2024-11-20T17:49:22,558 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 55a74b69bbfb418fb8c739c87d9b143b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:22,558 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:22,558 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=12, startTime=1732124962120; duration=0sec 2024-11-20T17:49:22,558 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:22,559 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:22,564 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:22,565 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/f8eac67cc8c84a2eb6a5d84b56121abb is 50, key is test_row_0/C:col10/1732124961247/Put/seqid=0 2024-11-20T17:49:22,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742176_1352 (size=13391) 2024-11-20T17:49:22,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125022567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36720 deadline: 1732125022658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,660 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,681 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5f717f779b7944478846cec7da6e6503 2024-11-20T17:49:22,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/07fb62a8a9ec4f97821b4d025322ede3 is 50, key is test_row_0/B:col10/1732124961293/Put/seqid=0 2024-11-20T17:49:22,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742177_1353 (size=12301) 2024-11-20T17:49:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36734 deadline: 1732125022708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,708 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36640 deadline: 1732125022747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,749 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in ee91b716d3c9cb4059bcd0deede2ec56 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18314 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:49:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125022767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125022778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:22,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:22,976 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/f8eac67cc8c84a2eb6a5d84b56121abb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f8eac67cc8c84a2eb6a5d84b56121abb 2024-11-20T17:49:22,982 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into f8eac67cc8c84a2eb6a5d84b56121abb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:22,982 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:22,983 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=12, startTime=1732124962120; duration=0sec 2024-11-20T17:49:22,983 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:22,983 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36684 deadline: 1732125023078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:23,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36666 deadline: 1732125023088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:49:23,101 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/07fb62a8a9ec4f97821b4d025322ede3 2024-11-20T17:49:23,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/9832d7a2e8254b03bf72b0ccb816b0ad is 50, key is test_row_0/C:col10/1732124961293/Put/seqid=0 2024-11-20T17:49:23,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742178_1354 (size=12301) 2024-11-20T17:49:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:23,519 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/9832d7a2e8254b03bf72b0ccb816b0ad 2024-11-20T17:49:23,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/5f717f779b7944478846cec7da6e6503 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5f717f779b7944478846cec7da6e6503 2024-11-20T17:49:23,532 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5f717f779b7944478846cec7da6e6503, entries=150, sequenceid=479, filesize=12.0 K 2024-11-20T17:49:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/07fb62a8a9ec4f97821b4d025322ede3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/07fb62a8a9ec4f97821b4d025322ede3 2024-11-20T17:49:23,537 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/07fb62a8a9ec4f97821b4d025322ede3, entries=150, sequenceid=479, filesize=12.0 K 2024-11-20T17:49:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/9832d7a2e8254b03bf72b0ccb816b0ad as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9832d7a2e8254b03bf72b0ccb816b0ad 2024-11-20T17:49:23,542 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9832d7a2e8254b03bf72b0ccb816b0ad, entries=150, sequenceid=479, filesize=12.0 K 2024-11-20T17:49:23,542 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ee91b716d3c9cb4059bcd0deede2ec56 in 1275ms, sequenceid=479, compaction requested=false 2024-11-20T17:49:23,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:23,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:23,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-20T17:49:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-20T17:49:23,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T17:49:23,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-11-20T17:49:23,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.1980 sec 2024-11-20T17:49:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:23,589 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:23,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:49:23,590 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-20T17:49:23,590 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:23,590 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:23,590 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:23,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:23,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/80cf75d42f804cfa9b5dc9c856edd55e is 50, key is test_row_0/A:col10/1732124962440/Put/seqid=0 2024-11-20T17:49:23,599 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-20T17:49:23,599 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:23,599 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:23,599 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:23,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:23,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742179_1355 (size=12301) 2024-11-20T17:49:24,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/80cf75d42f804cfa9b5dc9c856edd55e 2024-11-20T17:49:24,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/772670c26c4c42c1a9c6f28aedb0b240 is 50, key is test_row_0/B:col10/1732124962440/Put/seqid=0 2024-11-20T17:49:24,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742180_1356 (size=12301) 2024-11-20T17:49:24,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/772670c26c4c42c1a9c6f28aedb0b240 2024-11-20T17:49:24,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/338ce71cce1340caacce5a9b7cfb85e1 is 50, key is test_row_0/C:col10/1732124962440/Put/seqid=0 2024-11-20T17:49:24,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742181_1357 (size=12301) 2024-11-20T17:49:24,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/338ce71cce1340caacce5a9b7cfb85e1 2024-11-20T17:49:24,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/80cf75d42f804cfa9b5dc9c856edd55e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/80cf75d42f804cfa9b5dc9c856edd55e 2024-11-20T17:49:24,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/80cf75d42f804cfa9b5dc9c856edd55e, entries=150, sequenceid=494, filesize=12.0 K 2024-11-20T17:49:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/772670c26c4c42c1a9c6f28aedb0b240 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/772670c26c4c42c1a9c6f28aedb0b240 2024-11-20T17:49:24,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/772670c26c4c42c1a9c6f28aedb0b240, entries=150, sequenceid=494, filesize=12.0 K 2024-11-20T17:49:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/338ce71cce1340caacce5a9b7cfb85e1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/338ce71cce1340caacce5a9b7cfb85e1 2024-11-20T17:49:24,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/338ce71cce1340caacce5a9b7cfb85e1, entries=150, sequenceid=494, filesize=12.0 K 2024-11-20T17:49:24,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=6.71 KB/6870 for ee91b716d3c9cb4059bcd0deede2ec56 in 1258ms, sequenceid=494, compaction requested=true 2024-11-20T17:49:24,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:24,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:24,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:24,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:24,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:24,847 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:24,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee91b716d3c9cb4059bcd0deede2ec56:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:24,848 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:24,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:24,848 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:24,848 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:24,848 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/A is initiating minor compaction (all files) 2024-11-20T17:49:24,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/B is initiating minor compaction (all files) 2024-11-20T17:49:24,849 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/A in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:24,849 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/B in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:24,849 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/9a73604d34914040a25cbef052beacd1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5f717f779b7944478846cec7da6e6503, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/80cf75d42f804cfa9b5dc9c856edd55e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=37.1 K 2024-11-20T17:49:24,849 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/55a74b69bbfb418fb8c739c87d9b143b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/07fb62a8a9ec4f97821b4d025322ede3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/772670c26c4c42c1a9c6f28aedb0b240] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=37.1 K 2024-11-20T17:49:24,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a74b69bbfb418fb8c739c87d9b143b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732124959083 2024-11-20T17:49:24,849 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a73604d34914040a25cbef052beacd1, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732124959083 2024-11-20T17:49:24,849 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f717f779b7944478846cec7da6e6503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732124961268 2024-11-20T17:49:24,850 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 07fb62a8a9ec4f97821b4d025322ede3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732124961268 2024-11-20T17:49:24,850 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80cf75d42f804cfa9b5dc9c856edd55e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732124962440 2024-11-20T17:49:24,850 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 772670c26c4c42c1a9c6f28aedb0b240, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732124962440 2024-11-20T17:49:24,859 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#A#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:24,860 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/8828975c7432413abb178f8fe5c38a3e is 50, key is test_row_0/A:col10/1732124962440/Put/seqid=0 2024-11-20T17:49:24,862 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:24,863 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3ab8a480b8e54f2e9f61baae91007073 is 50, key is test_row_0/B:col10/1732124962440/Put/seqid=0 2024-11-20T17:49:24,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742182_1358 (size=13493) 2024-11-20T17:49:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742183_1359 (size=13493) 2024-11-20T17:49:25,269 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/8828975c7432413abb178f8fe5c38a3e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/8828975c7432413abb178f8fe5c38a3e 2024-11-20T17:49:25,271 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/3ab8a480b8e54f2e9f61baae91007073 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3ab8a480b8e54f2e9f61baae91007073 2024-11-20T17:49:25,274 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/A of ee91b716d3c9cb4059bcd0deede2ec56 into 8828975c7432413abb178f8fe5c38a3e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:25,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:25,274 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/A, priority=13, startTime=1732124964847; duration=0sec 2024-11-20T17:49:25,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:25,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:A 2024-11-20T17:49:25,274 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:25,274 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/B of ee91b716d3c9cb4059bcd0deede2ec56 into 3ab8a480b8e54f2e9f61baae91007073(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:25,274 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:25,274 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/B, priority=13, startTime=1732124964847; duration=0sec 2024-11-20T17:49:25,275 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:25,275 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:B 2024-11-20T17:49:25,275 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:25,275 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): ee91b716d3c9cb4059bcd0deede2ec56/C is initiating minor compaction (all files) 2024-11-20T17:49:25,275 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee91b716d3c9cb4059bcd0deede2ec56/C in TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:25,275 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f8eac67cc8c84a2eb6a5d84b56121abb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9832d7a2e8254b03bf72b0ccb816b0ad, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/338ce71cce1340caacce5a9b7cfb85e1] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp, totalSize=37.1 K 2024-11-20T17:49:25,275 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8eac67cc8c84a2eb6a5d84b56121abb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732124959083 2024-11-20T17:49:25,276 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9832d7a2e8254b03bf72b0ccb816b0ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732124961268 2024-11-20T17:49:25,276 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338ce71cce1340caacce5a9b7cfb85e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732124962440 2024-11-20T17:49:25,283 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee91b716d3c9cb4059bcd0deede2ec56#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:25,284 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/51b0c80741234e9d90aac5352101808f is 50, key is test_row_0/C:col10/1732124962440/Put/seqid=0 2024-11-20T17:49:25,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742184_1360 (size=13493) 2024-11-20T17:49:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T17:49:25,508 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:25,693 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/51b0c80741234e9d90aac5352101808f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/51b0c80741234e9d90aac5352101808f 2024-11-20T17:49:25,698 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee91b716d3c9cb4059bcd0deede2ec56/C of ee91b716d3c9cb4059bcd0deede2ec56 into 51b0c80741234e9d90aac5352101808f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:25,698 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee91b716d3c9cb4059bcd0deede2ec56: 2024-11-20T17:49:25,699 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., storeName=ee91b716d3c9cb4059bcd0deede2ec56/C, priority=13, startTime=1732124964847; duration=0sec 2024-11-20T17:49:25,699 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:25,699 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee91b716d3c9cb4059bcd0deede2ec56:C 2024-11-20T17:49:32,709 INFO [Thread-1171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1171. 2024-11-20T17:49:32,709 DEBUG [Thread-1171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:32,709 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,710 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:32,759 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-20T17:49:32,759 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:32,759 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,759 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:32,850 INFO [Thread-1173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1173. 2024-11-20T17:49:32,851 DEBUG [Thread-1173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:32,851 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,851 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3877 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3842 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3722 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3904 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3843 2024-11-20T17:49:32,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T17:49:32,851 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:49:32,851 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:49:32,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:32,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:32,852 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T17:49:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T17:49:32,855 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124972855"}]},"ts":"1732124972855"} 2024-11-20T17:49:32,857 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T17:49:32,857 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T17:49:32,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:49:32,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, UNASSIGN}] 2024-11-20T17:49:32,863 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, UNASSIGN 2024-11-20T17:49:32,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ee91b716d3c9cb4059bcd0deede2ec56, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:32,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, UNASSIGN because future has completed 2024-11-20T17:49:32,875 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:49:32,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:49:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T17:49:33,028 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:33,028 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:49:33,028 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing ee91b716d3c9cb4059bcd0deede2ec56, disabling compactions & flushes 2024-11-20T17:49:33,028 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:33,028 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:33,028 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. after waiting 0 ms 2024-11-20T17:49:33,028 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:33,029 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing ee91b716d3c9cb4059bcd0deede2ec56 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-20T17:49:33,032 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/2a7eae4a27e14f63a8f3e2da0d53094a is 50, key is test_row_0/A:col10/1732124972708/Put/seqid=0 2024-11-20T17:49:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742185_1361 (size=12301) 2024-11-20T17:49:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T17:49:33,436 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/2a7eae4a27e14f63a8f3e2da0d53094a 2024-11-20T17:49:33,443 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/819d8f4e927b4025ac19ac3ad9438fe8 is 50, key is test_row_0/B:col10/1732124972708/Put/seqid=0 2024-11-20T17:49:33,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742186_1362 (size=12301) 2024-11-20T17:49:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T17:49:33,846 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/819d8f4e927b4025ac19ac3ad9438fe8 2024-11-20T17:49:33,853 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/13487abfbf35433f84c2b2d312a55825 is 50, key is test_row_0/C:col10/1732124972708/Put/seqid=0 2024-11-20T17:49:33,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742187_1363 (size=12301) 2024-11-20T17:49:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T17:49:34,269 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/13487abfbf35433f84c2b2d312a55825 2024-11-20T17:49:34,275 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/A/2a7eae4a27e14f63a8f3e2da0d53094a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2a7eae4a27e14f63a8f3e2da0d53094a 2024-11-20T17:49:34,280 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2a7eae4a27e14f63a8f3e2da0d53094a, entries=150, sequenceid=504, filesize=12.0 K 2024-11-20T17:49:34,281 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/B/819d8f4e927b4025ac19ac3ad9438fe8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/819d8f4e927b4025ac19ac3ad9438fe8 2024-11-20T17:49:34,285 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/819d8f4e927b4025ac19ac3ad9438fe8, entries=150, sequenceid=504, filesize=12.0 K 2024-11-20T17:49:34,286 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/.tmp/C/13487abfbf35433f84c2b2d312a55825 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/13487abfbf35433f84c2b2d312a55825 2024-11-20T17:49:34,291 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/13487abfbf35433f84c2b2d312a55825, entries=150, sequenceid=504, filesize=12.0 K 2024-11-20T17:49:34,292 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for ee91b716d3c9cb4059bcd0deede2ec56 in 1264ms, sequenceid=504, compaction requested=false 2024-11-20T17:49:34,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0944e20c2c5643d79acdea830063310a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/35ff967c1ef744598cead4e8a86af702, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/338feb8912714ca1bda468982ae946d0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f1c0469465f94162b0576f5d05ff2dfa, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/086bdf1421b44aa282c46396c262a794, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/a07ce84d28034f69b198fae03f24749d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/e012921b4156499bb5a18813eef90357, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/b9f838f493db448d8cde842e7dd42eea, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d3a961bd91004d0085af2cb168039eed, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f936ee51c5134c79bf2bb71c8f580a3e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5017fbb663094ea69e2257855e2ae6b9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1e0f1f9a09ac48559fce19fbe8a5b5d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/fe726db31d5145d89510be8f2ddb5190, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d654ff20a29246be8a07c2122a0e1a83, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0a7146134d9e42cdbbeea19708eb753c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/36edd9d8d7384a67a40be0e93dc38e41, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/62c58e98b9a1488ca6b7055dc865fb13, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/4ae4907e37da49539654e53a85464ec3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/17de4063cdc84528aaebcef1159b1a58, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/91dfc5f1d493446a8ea00684e2c385cc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/31500503edff46dea00c9a7e1eb07d19, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/cc4ee0d99f054e7c8058edc2667d033f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1383d6620e5c4e749f75da5f86b854d3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/7c6534ec40fd4dd6b54ca4ad9a89067e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/815dd3f509ba4ed296960923f6a3061a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/24662350dac041cd8cb9626fb8d82710, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f8d64b38c7ce4cb9991bce4b72528c05, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5ba9879965b64223a30a0c7a528c5347, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d739a29c491f428fb544dcae7d8ba8c5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f23c3894a36e4fb78811177b51a67116, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/9a73604d34914040a25cbef052beacd1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/902f88ee1a974d42b4f8fa6666ee8aed, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5f717f779b7944478846cec7da6e6503, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/80cf75d42f804cfa9b5dc9c856edd55e] to archive 2024-11-20T17:49:34,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:34,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0944e20c2c5643d79acdea830063310a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0944e20c2c5643d79acdea830063310a 2024-11-20T17:49:34,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/35ff967c1ef744598cead4e8a86af702 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/35ff967c1ef744598cead4e8a86af702 2024-11-20T17:49:34,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2aa8ea6f30f14ed7824cfc0c9f2ffcae 2024-11-20T17:49:34,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/338feb8912714ca1bda468982ae946d0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/338feb8912714ca1bda468982ae946d0 2024-11-20T17:49:34,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f1c0469465f94162b0576f5d05ff2dfa to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f1c0469465f94162b0576f5d05ff2dfa 2024-11-20T17:49:34,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/086bdf1421b44aa282c46396c262a794 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/086bdf1421b44aa282c46396c262a794 2024-11-20T17:49:34,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/a07ce84d28034f69b198fae03f24749d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/a07ce84d28034f69b198fae03f24749d 2024-11-20T17:49:34,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/e012921b4156499bb5a18813eef90357 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/e012921b4156499bb5a18813eef90357 2024-11-20T17:49:34,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/b9f838f493db448d8cde842e7dd42eea to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/b9f838f493db448d8cde842e7dd42eea 2024-11-20T17:49:34,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d3a961bd91004d0085af2cb168039eed to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d3a961bd91004d0085af2cb168039eed 2024-11-20T17:49:34,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f936ee51c5134c79bf2bb71c8f580a3e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f936ee51c5134c79bf2bb71c8f580a3e 2024-11-20T17:49:34,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5017fbb663094ea69e2257855e2ae6b9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5017fbb663094ea69e2257855e2ae6b9 2024-11-20T17:49:34,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1e0f1f9a09ac48559fce19fbe8a5b5d9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1e0f1f9a09ac48559fce19fbe8a5b5d9 2024-11-20T17:49:34,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/fe726db31d5145d89510be8f2ddb5190 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/fe726db31d5145d89510be8f2ddb5190 2024-11-20T17:49:34,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d654ff20a29246be8a07c2122a0e1a83 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d654ff20a29246be8a07c2122a0e1a83 2024-11-20T17:49:34,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0a7146134d9e42cdbbeea19708eb753c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/0a7146134d9e42cdbbeea19708eb753c 2024-11-20T17:49:34,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/36edd9d8d7384a67a40be0e93dc38e41 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/36edd9d8d7384a67a40be0e93dc38e41 2024-11-20T17:49:34,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/62c58e98b9a1488ca6b7055dc865fb13 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/62c58e98b9a1488ca6b7055dc865fb13 2024-11-20T17:49:34,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/4ae4907e37da49539654e53a85464ec3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/4ae4907e37da49539654e53a85464ec3 2024-11-20T17:49:34,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/17de4063cdc84528aaebcef1159b1a58 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/17de4063cdc84528aaebcef1159b1a58 2024-11-20T17:49:34,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/91dfc5f1d493446a8ea00684e2c385cc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/91dfc5f1d493446a8ea00684e2c385cc 2024-11-20T17:49:34,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/31500503edff46dea00c9a7e1eb07d19 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/31500503edff46dea00c9a7e1eb07d19 2024-11-20T17:49:34,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/cc4ee0d99f054e7c8058edc2667d033f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/cc4ee0d99f054e7c8058edc2667d033f 2024-11-20T17:49:34,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1383d6620e5c4e749f75da5f86b854d3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/1383d6620e5c4e749f75da5f86b854d3 2024-11-20T17:49:34,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/7c6534ec40fd4dd6b54ca4ad9a89067e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/7c6534ec40fd4dd6b54ca4ad9a89067e 2024-11-20T17:49:34,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/815dd3f509ba4ed296960923f6a3061a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/815dd3f509ba4ed296960923f6a3061a 2024-11-20T17:49:34,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/24662350dac041cd8cb9626fb8d82710 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/24662350dac041cd8cb9626fb8d82710 2024-11-20T17:49:34,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f8d64b38c7ce4cb9991bce4b72528c05 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f8d64b38c7ce4cb9991bce4b72528c05 2024-11-20T17:49:34,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5ba9879965b64223a30a0c7a528c5347 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5ba9879965b64223a30a0c7a528c5347 2024-11-20T17:49:34,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d739a29c491f428fb544dcae7d8ba8c5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/d739a29c491f428fb544dcae7d8ba8c5 2024-11-20T17:49:34,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f23c3894a36e4fb78811177b51a67116 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/f23c3894a36e4fb78811177b51a67116 2024-11-20T17:49:34,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/9a73604d34914040a25cbef052beacd1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/9a73604d34914040a25cbef052beacd1 2024-11-20T17:49:34,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/902f88ee1a974d42b4f8fa6666ee8aed to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/902f88ee1a974d42b4f8fa6666ee8aed 2024-11-20T17:49:34,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5f717f779b7944478846cec7da6e6503 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/5f717f779b7944478846cec7da6e6503 2024-11-20T17:49:34,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/80cf75d42f804cfa9b5dc9c856edd55e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/80cf75d42f804cfa9b5dc9c856edd55e 2024-11-20T17:49:34,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6be40bdf97034519803ba1d0048fa865, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce81974552ae421689d3acf3961e4fe9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/4133315609da4cc5aebe5bc2e383a632, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2252a34433d74394a0820d131dde8107, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/e1e089a2bb154e1d9ec50993a003775b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/d53d3e7b6cb64b438cf064eb4013724c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9e930ae201ce42d398b4ddea0814885e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce77f6def5344d1f9669b808de4d8c93, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/731c4e2760714577b7129ffb860386f8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ba2fb363d0c444d1bb75907ee6e06be2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3d99d4ac0ebd47238ed61d0c11dc0ccf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a13d6e317dd840df914bed60428ad16e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/7db57801e8b64e7cb0bb4ed336879f25, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/71d0371f689d4c12a1601d69208feedd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/91c82e65443d45089e4fbc33173dc0b3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2c3f3446bc8d43b3836f33546517789b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6fae00262df140329aa351f6081c66c4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3319d6200180480385c638af6f5df3ae, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/23329ca01bce4847bb1831264db671ed, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/cb976ddfb3554dc292c1276453bc007b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/8809f20752ca461b85a7ae847224d347, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a0b39af7e81943939204409d9a2c40e3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/b1739eb283524115a7961aa2b9c392e2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/efcdd6c21f514de4b9baf863aecbc000, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/aefbb8bad637473a9f54d5e81af3c4b8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9c2109ca4cef4d6997ff7f7ae83d6e4e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/f3126f3965ab41199203a0d28d0271d7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/938f80bd70fe43e9a2dd04e31fffd950, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/95ae73502e23483495a0b80f26d08391, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/33e8b1d3c20c4128893503a89cb101d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3c060daec37e40bd8b87ff6d95cb94c0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/55a74b69bbfb418fb8c739c87d9b143b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/0ef95fe888cd4f9cb5ae859147f14b30, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/07fb62a8a9ec4f97821b4d025322ede3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/772670c26c4c42c1a9c6f28aedb0b240] to archive 2024-11-20T17:49:34,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:34,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6be40bdf97034519803ba1d0048fa865 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6be40bdf97034519803ba1d0048fa865 2024-11-20T17:49:34,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce81974552ae421689d3acf3961e4fe9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce81974552ae421689d3acf3961e4fe9 2024-11-20T17:49:34,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/4133315609da4cc5aebe5bc2e383a632 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/4133315609da4cc5aebe5bc2e383a632 2024-11-20T17:49:34,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2252a34433d74394a0820d131dde8107 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2252a34433d74394a0820d131dde8107 2024-11-20T17:49:34,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/e1e089a2bb154e1d9ec50993a003775b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/e1e089a2bb154e1d9ec50993a003775b 2024-11-20T17:49:34,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/d53d3e7b6cb64b438cf064eb4013724c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/d53d3e7b6cb64b438cf064eb4013724c 2024-11-20T17:49:34,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9e930ae201ce42d398b4ddea0814885e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9e930ae201ce42d398b4ddea0814885e 2024-11-20T17:49:34,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce77f6def5344d1f9669b808de4d8c93 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ce77f6def5344d1f9669b808de4d8c93 2024-11-20T17:49:34,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/731c4e2760714577b7129ffb860386f8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/731c4e2760714577b7129ffb860386f8 2024-11-20T17:49:34,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ba2fb363d0c444d1bb75907ee6e06be2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/ba2fb363d0c444d1bb75907ee6e06be2 2024-11-20T17:49:34,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3d99d4ac0ebd47238ed61d0c11dc0ccf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3d99d4ac0ebd47238ed61d0c11dc0ccf 2024-11-20T17:49:34,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a13d6e317dd840df914bed60428ad16e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a13d6e317dd840df914bed60428ad16e 2024-11-20T17:49:34,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/7db57801e8b64e7cb0bb4ed336879f25 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/7db57801e8b64e7cb0bb4ed336879f25 2024-11-20T17:49:34,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/71d0371f689d4c12a1601d69208feedd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/71d0371f689d4c12a1601d69208feedd 2024-11-20T17:49:34,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/91c82e65443d45089e4fbc33173dc0b3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/91c82e65443d45089e4fbc33173dc0b3 2024-11-20T17:49:34,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2c3f3446bc8d43b3836f33546517789b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/2c3f3446bc8d43b3836f33546517789b 2024-11-20T17:49:34,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6fae00262df140329aa351f6081c66c4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/6fae00262df140329aa351f6081c66c4 2024-11-20T17:49:34,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3319d6200180480385c638af6f5df3ae to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3319d6200180480385c638af6f5df3ae 2024-11-20T17:49:34,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/23329ca01bce4847bb1831264db671ed to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/23329ca01bce4847bb1831264db671ed 2024-11-20T17:49:34,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/cb976ddfb3554dc292c1276453bc007b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/cb976ddfb3554dc292c1276453bc007b 2024-11-20T17:49:34,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/8809f20752ca461b85a7ae847224d347 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/8809f20752ca461b85a7ae847224d347 2024-11-20T17:49:34,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a0b39af7e81943939204409d9a2c40e3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/a0b39af7e81943939204409d9a2c40e3 2024-11-20T17:49:34,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/b1739eb283524115a7961aa2b9c392e2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/b1739eb283524115a7961aa2b9c392e2 2024-11-20T17:49:34,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/efcdd6c21f514de4b9baf863aecbc000 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/efcdd6c21f514de4b9baf863aecbc000 2024-11-20T17:49:34,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/aefbb8bad637473a9f54d5e81af3c4b8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/aefbb8bad637473a9f54d5e81af3c4b8 2024-11-20T17:49:34,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9c2109ca4cef4d6997ff7f7ae83d6e4e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/9c2109ca4cef4d6997ff7f7ae83d6e4e 2024-11-20T17:49:34,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/f3126f3965ab41199203a0d28d0271d7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/f3126f3965ab41199203a0d28d0271d7 2024-11-20T17:49:34,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/938f80bd70fe43e9a2dd04e31fffd950 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/938f80bd70fe43e9a2dd04e31fffd950 2024-11-20T17:49:34,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/95ae73502e23483495a0b80f26d08391 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/95ae73502e23483495a0b80f26d08391 2024-11-20T17:49:34,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/33e8b1d3c20c4128893503a89cb101d9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/33e8b1d3c20c4128893503a89cb101d9 2024-11-20T17:49:34,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3c060daec37e40bd8b87ff6d95cb94c0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3c060daec37e40bd8b87ff6d95cb94c0 2024-11-20T17:49:34,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/55a74b69bbfb418fb8c739c87d9b143b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/55a74b69bbfb418fb8c739c87d9b143b 2024-11-20T17:49:34,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/0ef95fe888cd4f9cb5ae859147f14b30 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/0ef95fe888cd4f9cb5ae859147f14b30 2024-11-20T17:49:34,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/07fb62a8a9ec4f97821b4d025322ede3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/07fb62a8a9ec4f97821b4d025322ede3 2024-11-20T17:49:34,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/772670c26c4c42c1a9c6f28aedb0b240 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/772670c26c4c42c1a9c6f28aedb0b240 2024-11-20T17:49:34,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/22c097b1df6640588e007f1af3701b2e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/2512df9f1d494d748b12a7ac2fcad4e3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f52611722c8247b9abe07af419a2ae78, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/89af299303f143f7bf395c8d585a37ff, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fa3fb2b67ae748749b6edd332d0c0097, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/d52aa2764d6b4fb5bc59b7e6bf5170a7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/8252587a3fbe41d3b79dbc18e3770bc9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/5bad65c1018b42ea8f22adb67cda1be4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/4ce6dbb7e8ca4240a02222ffd60f71e5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/dd673359af2f4e0bbfca9004cd3a0148, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/11ff9620637f412b9aeefa57734168f1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/ebc5352d716b47f2acdb2d90c5847c18, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/549f140ffa974633b6b5f76417ab392e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/7534e84a14bd49edb2dfd217fc6d95bb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/08ec618d6ff54cbea90c47e21f148e49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/412ee5258fc14ba99ffafa643d2e6647, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/44bf9b52003744db8626a26ef65e8c1a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/0071b39db9724b1f84c5d17f8d3841cd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06fa114e00344965b644ac05e076aabd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b6de0457fe3143d6a5afec1f8efa4d2c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/398f54a7aa684674a7d87a4a5dc5676a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06328f5496804a9993ca9f573389ea85, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9d4a0741dbd74a65b810ea4192ee40d2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/50ba5a408ca445cf89678d94fbe534a0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b34b50f272ac48cead2460e0aec75cb4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fc4ad5410ddd493f9c8f3d935ad3cf31, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19f2d4cc9aca485f91d764b4b9e91461, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19698cc527eb4b5c8752e8583f35b66d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b4a939a8a65044e7bf1c7193c993cb7a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/565ae948fb4e46e5a796dc66c68b9943, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/a281b586532946a4b3ca9d3e588a9c46, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f8eac67cc8c84a2eb6a5d84b56121abb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/060f5fbc42414c4dae541f80b9b9c156, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9832d7a2e8254b03bf72b0ccb816b0ad, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/338ce71cce1340caacce5a9b7cfb85e1] to archive 2024-11-20T17:49:34,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:34,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/22c097b1df6640588e007f1af3701b2e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/22c097b1df6640588e007f1af3701b2e 2024-11-20T17:49:34,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/2512df9f1d494d748b12a7ac2fcad4e3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/2512df9f1d494d748b12a7ac2fcad4e3 2024-11-20T17:49:34,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f52611722c8247b9abe07af419a2ae78 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f52611722c8247b9abe07af419a2ae78 2024-11-20T17:49:34,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/89af299303f143f7bf395c8d585a37ff to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/89af299303f143f7bf395c8d585a37ff 2024-11-20T17:49:34,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fa3fb2b67ae748749b6edd332d0c0097 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fa3fb2b67ae748749b6edd332d0c0097 2024-11-20T17:49:34,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/d52aa2764d6b4fb5bc59b7e6bf5170a7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/d52aa2764d6b4fb5bc59b7e6bf5170a7 2024-11-20T17:49:34,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/8252587a3fbe41d3b79dbc18e3770bc9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/8252587a3fbe41d3b79dbc18e3770bc9 2024-11-20T17:49:34,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/5bad65c1018b42ea8f22adb67cda1be4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/5bad65c1018b42ea8f22adb67cda1be4 2024-11-20T17:49:34,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/4ce6dbb7e8ca4240a02222ffd60f71e5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/4ce6dbb7e8ca4240a02222ffd60f71e5 2024-11-20T17:49:34,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/dd673359af2f4e0bbfca9004cd3a0148 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/dd673359af2f4e0bbfca9004cd3a0148 2024-11-20T17:49:34,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/11ff9620637f412b9aeefa57734168f1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/11ff9620637f412b9aeefa57734168f1 2024-11-20T17:49:34,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/ebc5352d716b47f2acdb2d90c5847c18 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/ebc5352d716b47f2acdb2d90c5847c18 2024-11-20T17:49:34,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/549f140ffa974633b6b5f76417ab392e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/549f140ffa974633b6b5f76417ab392e 2024-11-20T17:49:34,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/7534e84a14bd49edb2dfd217fc6d95bb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/7534e84a14bd49edb2dfd217fc6d95bb 2024-11-20T17:49:34,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/08ec618d6ff54cbea90c47e21f148e49 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/08ec618d6ff54cbea90c47e21f148e49 2024-11-20T17:49:34,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/412ee5258fc14ba99ffafa643d2e6647 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/412ee5258fc14ba99ffafa643d2e6647 2024-11-20T17:49:34,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/44bf9b52003744db8626a26ef65e8c1a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/44bf9b52003744db8626a26ef65e8c1a 2024-11-20T17:49:34,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/0071b39db9724b1f84c5d17f8d3841cd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/0071b39db9724b1f84c5d17f8d3841cd 2024-11-20T17:49:34,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06fa114e00344965b644ac05e076aabd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06fa114e00344965b644ac05e076aabd 2024-11-20T17:49:34,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b6de0457fe3143d6a5afec1f8efa4d2c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b6de0457fe3143d6a5afec1f8efa4d2c 2024-11-20T17:49:34,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/398f54a7aa684674a7d87a4a5dc5676a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/398f54a7aa684674a7d87a4a5dc5676a 2024-11-20T17:49:34,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06328f5496804a9993ca9f573389ea85 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/06328f5496804a9993ca9f573389ea85 2024-11-20T17:49:34,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9d4a0741dbd74a65b810ea4192ee40d2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9d4a0741dbd74a65b810ea4192ee40d2 2024-11-20T17:49:34,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/50ba5a408ca445cf89678d94fbe534a0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/50ba5a408ca445cf89678d94fbe534a0 2024-11-20T17:49:34,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b34b50f272ac48cead2460e0aec75cb4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b34b50f272ac48cead2460e0aec75cb4 2024-11-20T17:49:34,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fc4ad5410ddd493f9c8f3d935ad3cf31 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/fc4ad5410ddd493f9c8f3d935ad3cf31 2024-11-20T17:49:34,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19f2d4cc9aca485f91d764b4b9e91461 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19f2d4cc9aca485f91d764b4b9e91461 2024-11-20T17:49:34,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19698cc527eb4b5c8752e8583f35b66d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/19698cc527eb4b5c8752e8583f35b66d 2024-11-20T17:49:34,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b4a939a8a65044e7bf1c7193c993cb7a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/b4a939a8a65044e7bf1c7193c993cb7a 2024-11-20T17:49:34,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/565ae948fb4e46e5a796dc66c68b9943 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/565ae948fb4e46e5a796dc66c68b9943 2024-11-20T17:49:34,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/a281b586532946a4b3ca9d3e588a9c46 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/a281b586532946a4b3ca9d3e588a9c46 2024-11-20T17:49:34,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f8eac67cc8c84a2eb6a5d84b56121abb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/f8eac67cc8c84a2eb6a5d84b56121abb 2024-11-20T17:49:34,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/060f5fbc42414c4dae541f80b9b9c156 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/060f5fbc42414c4dae541f80b9b9c156 2024-11-20T17:49:34,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9832d7a2e8254b03bf72b0ccb816b0ad to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/9832d7a2e8254b03bf72b0ccb816b0ad 2024-11-20T17:49:34,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/338ce71cce1340caacce5a9b7cfb85e1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/338ce71cce1340caacce5a9b7cfb85e1 2024-11-20T17:49:34,427 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/recovered.edits/507.seqid, newMaxSeqId=507, maxSeqId=1 2024-11-20T17:49:34,428 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. 2024-11-20T17:49:34,428 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for ee91b716d3c9cb4059bcd0deede2ec56: Waiting for close lock at 1732124973028Running coprocessor pre-close hooks at 1732124973028Disabling compacts and flushes for region at 1732124973028Disabling writes for close at 1732124973028Obtaining lock to block concurrent updates at 1732124973029 (+1 ms)Preparing flush snapshotting stores in ee91b716d3c9cb4059bcd0deede2ec56 at 1732124973029Finished memstore snapshotting TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732124973029Flushing stores of TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56. at 1732124973029Flushing ee91b716d3c9cb4059bcd0deede2ec56/A: creating writer at 1732124973029Flushing ee91b716d3c9cb4059bcd0deede2ec56/A: appending metadata at 1732124973032 (+3 ms)Flushing ee91b716d3c9cb4059bcd0deede2ec56/A: closing flushed file at 1732124973032Flushing ee91b716d3c9cb4059bcd0deede2ec56/B: creating writer at 1732124973440 (+408 ms)Flushing ee91b716d3c9cb4059bcd0deede2ec56/B: appending metadata at 1732124973442 (+2 ms)Flushing ee91b716d3c9cb4059bcd0deede2ec56/B: closing flushed file at 1732124973442Flushing ee91b716d3c9cb4059bcd0deede2ec56/C: creating writer at 1732124973850 (+408 ms)Flushing ee91b716d3c9cb4059bcd0deede2ec56/C: appending metadata at 1732124973852 (+2 ms)Flushing ee91b716d3c9cb4059bcd0deede2ec56/C: closing flushed file at 1732124973852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa151e5: reopening flushed file at 1732124974274 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6ddb0b: reopening flushed file at 1732124974280 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42d79b9c: reopening flushed file at 1732124974286 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for ee91b716d3c9cb4059bcd0deede2ec56 in 1264ms, sequenceid=504, compaction requested=false at 1732124974292 (+6 ms)Writing region close event to WAL at 1732124974425 (+133 ms)Running coprocessor post-close hooks at 1732124974428 (+3 ms)Closed at 1732124974428 2024-11-20T17:49:34,430 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:34,430 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ee91b716d3c9cb4059bcd0deede2ec56, regionState=CLOSED 2024-11-20T17:49:34,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:49:34,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-20T17:49:34,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure ee91b716d3c9cb4059bcd0deede2ec56, server=613581fcdc40,44957,1732124872893 in 1.5580 sec 2024-11-20T17:49:34,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-20T17:49:34,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee91b716d3c9cb4059bcd0deede2ec56, UNASSIGN in 1.5730 sec 2024-11-20T17:49:34,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-20T17:49:34,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5780 sec 2024-11-20T17:49:34,438 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124974438"}]},"ts":"1732124974438"} 2024-11-20T17:49:34,440 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T17:49:34,440 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T17:49:34,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5880 sec 2024-11-20T17:49:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T17:49:34,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:34,998 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T17:49:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,000 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T17:49:35,001 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,004 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:35,005 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/recovered.edits] 2024-11-20T17:49:35,008 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2a7eae4a27e14f63a8f3e2da0d53094a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/2a7eae4a27e14f63a8f3e2da0d53094a 2024-11-20T17:49:35,009 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/8828975c7432413abb178f8fe5c38a3e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/A/8828975c7432413abb178f8fe5c38a3e 2024-11-20T17:49:35,011 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3ab8a480b8e54f2e9f61baae91007073 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/3ab8a480b8e54f2e9f61baae91007073 2024-11-20T17:49:35,011 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/819d8f4e927b4025ac19ac3ad9438fe8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/B/819d8f4e927b4025ac19ac3ad9438fe8 2024-11-20T17:49:35,013 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/13487abfbf35433f84c2b2d312a55825 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/13487abfbf35433f84c2b2d312a55825 2024-11-20T17:49:35,013 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/51b0c80741234e9d90aac5352101808f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/C/51b0c80741234e9d90aac5352101808f 2024-11-20T17:49:35,016 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/recovered.edits/507.seqid to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56/recovered.edits/507.seqid 2024-11-20T17:49:35,016 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/ee91b716d3c9cb4059bcd0deede2ec56 2024-11-20T17:49:35,016 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T17:49:35,018 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,022 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T17:49:35,024 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T17:49:35,025 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,025 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T17:49:35,026 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732124975025"}]},"ts":"9223372036854775807"} 2024-11-20T17:49:35,027 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T17:49:35,027 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee91b716d3c9cb4059bcd0deede2ec56, NAME => 'TestAcidGuarantees,,1732124941845.ee91b716d3c9cb4059bcd0deede2ec56.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T17:49:35,028 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T17:49:35,028 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732124975028"}]},"ts":"9223372036854775807"} 2024-11-20T17:49:35,030 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T17:49:35,031 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-20T17:49:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T17:49:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T17:49:35,111 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:35,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=236 (was 238), OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=7917 (was 9109) 2024-11-20T17:49:35,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=236, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=7917 2024-11-20T17:49:35,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:49:35,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:49:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:35,136 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:49:35,136 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:35,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-20T17:49:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T17:49:35,137 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:49:35,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742188_1364 (size=959) 2024-11-20T17:49:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T17:49:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T17:49:35,545 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:49:35,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742189_1365 (size=53) 2024-11-20T17:49:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T17:49:35,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:49:35,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d16d3ef4696b9dba5a878aefd3c902e4, disabling compactions & flushes 2024-11-20T17:49:35,951 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:35,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:35,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. after waiting 0 ms 2024-11-20T17:49:35,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:35,951 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:35,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d16d3ef4696b9dba5a878aefd3c902e4: Waiting for close lock at 1732124975951Disabling compacts and flushes for region at 1732124975951Disabling writes for close at 1732124975951Writing region close event to WAL at 1732124975951Closed at 1732124975951 2024-11-20T17:49:35,954 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:49:35,954 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732124975954"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732124975954"}]},"ts":"1732124975954"} 2024-11-20T17:49:35,957 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:49:35,958 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:49:35,959 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124975958"}]},"ts":"1732124975958"} 2024-11-20T17:49:35,962 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T17:49:35,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, ASSIGN}] 2024-11-20T17:49:35,964 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, ASSIGN 2024-11-20T17:49:35,965 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, ASSIGN; state=OFFLINE, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=false 2024-11-20T17:49:36,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:36,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, ASSIGN because future has completed 2024-11-20T17:49:36,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:49:36,275 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:36,275 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:49:36,276 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,276 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:49:36,276 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,276 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T17:49:36,277 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,278 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d3ef4696b9dba5a878aefd3c902e4 columnFamilyName A 2024-11-20T17:49:36,278 DEBUG [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:36,279 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(327): Store=d16d3ef4696b9dba5a878aefd3c902e4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:36,279 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,280 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d3ef4696b9dba5a878aefd3c902e4 columnFamilyName B 2024-11-20T17:49:36,280 DEBUG [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:36,280 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(327): Store=d16d3ef4696b9dba5a878aefd3c902e4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:36,280 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,281 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d3ef4696b9dba5a878aefd3c902e4 columnFamilyName C 2024-11-20T17:49:36,281 DEBUG [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:36,282 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(327): Store=d16d3ef4696b9dba5a878aefd3c902e4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:36,282 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,282 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,283 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,283 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,283 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,284 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:49:36,284 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,286 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:49:36,286 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened d16d3ef4696b9dba5a878aefd3c902e4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71537835, jitterRate=0.06599681079387665}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:49:36,286 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:36,287 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for d16d3ef4696b9dba5a878aefd3c902e4: Running coprocessor pre-open hook at 1732124976276Writing region info on filesystem at 1732124976276Initializing all the Stores at 1732124976277 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124976277Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124976277Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124976277Cleaning up temporary data from old regions at 1732124976283 (+6 ms)Running coprocessor post-open hooks at 1732124976286 (+3 ms)Region opened successfully at 1732124976287 (+1 ms) 2024-11-20T17:49:36,287 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., pid=97, masterSystemTime=1732124976272 2024-11-20T17:49:36,289 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:36,289 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:36,289 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:36,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:49:36,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-20T17:49:36,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 in 172 msec 2024-11-20T17:49:36,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-20T17:49:36,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, ASSIGN in 331 msec 2024-11-20T17:49:36,295 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:49:36,295 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732124976295"}]},"ts":"1732124976295"} 2024-11-20T17:49:36,296 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T17:49:36,297 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:49:36,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-20T17:49:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T17:49:37,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:37,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29820261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:37,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:37,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:37,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c83b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:37,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:37,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fe9b78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:37,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:37,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:37,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:37,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:49:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:49:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2cd189df 2024-11-20T17:49:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:49:37,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:49:37,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:49:37,301 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:49:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T17:49:37,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742190_1366 (size=995) 2024-11-20T17:49:37,715 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T17:49:37,715 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T17:49:37,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:49:37,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, REOPEN/MOVE}] 2024-11-20T17:49:37,723 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, REOPEN/MOVE 2024-11-20T17:49:37,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:37,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, REOPEN/MOVE because future has completed 2024-11-20T17:49:37,727 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:49:37,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:49:37,880 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:37,880 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:49:37,880 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing d16d3ef4696b9dba5a878aefd3c902e4, disabling compactions & flushes 2024-11-20T17:49:37,880 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:37,880 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:37,880 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. after waiting 0 ms 2024-11-20T17:49:37,880 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:37,886 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:49:37,887 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:37,887 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for d16d3ef4696b9dba5a878aefd3c902e4: Waiting for close lock at 1732124977880Running coprocessor pre-close hooks at 1732124977880Disabling compacts and flushes for region at 1732124977880Disabling writes for close at 1732124977880Writing region close event to WAL at 1732124977881 (+1 ms)Running coprocessor post-close hooks at 1732124977887 (+6 ms)Closed at 1732124977887 2024-11-20T17:49:37,887 WARN [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: d16d3ef4696b9dba5a878aefd3c902e4 to self. 2024-11-20T17:49:37,889 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:37,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=CLOSED 2024-11-20T17:49:37,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:49:37,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-20T17:49:37,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 in 166 msec 2024-11-20T17:49:37,896 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, REOPEN/MOVE; state=CLOSED, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=true 2024-11-20T17:49:38,047 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, REOPEN/MOVE because future has completed 2024-11-20T17:49:38,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:49:38,204 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,204 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:49:38,204 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,204 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:49:38,205 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,205 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,206 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,207 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d3ef4696b9dba5a878aefd3c902e4 columnFamilyName A 2024-11-20T17:49:38,208 DEBUG [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:38,208 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(327): Store=d16d3ef4696b9dba5a878aefd3c902e4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:38,209 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,210 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d3ef4696b9dba5a878aefd3c902e4 columnFamilyName B 2024-11-20T17:49:38,210 DEBUG [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:38,210 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(327): Store=d16d3ef4696b9dba5a878aefd3c902e4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:38,210 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,211 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d3ef4696b9dba5a878aefd3c902e4 columnFamilyName C 2024-11-20T17:49:38,211 DEBUG [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:38,211 INFO [StoreOpener-d16d3ef4696b9dba5a878aefd3c902e4-1 {}] regionserver.HStore(327): Store=d16d3ef4696b9dba5a878aefd3c902e4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:49:38,211 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,212 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,213 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,214 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,214 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,214 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:49:38,215 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,216 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened d16d3ef4696b9dba5a878aefd3c902e4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63661754, jitterRate=-0.051365941762924194}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:49:38,216 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,217 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for d16d3ef4696b9dba5a878aefd3c902e4: Running coprocessor pre-open hook at 1732124978205Writing region info on filesystem at 1732124978205Initializing all the Stores at 1732124978205Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124978206 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124978206Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732124978206Cleaning up temporary data from old regions at 1732124978214 (+8 ms)Running coprocessor post-open hooks at 1732124978216 (+2 ms)Region opened successfully at 1732124978217 (+1 ms) 2024-11-20T17:49:38,218 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., pid=102, masterSystemTime=1732124978201 2024-11-20T17:49:38,219 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,219 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,220 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=OPEN, openSeqNum=5, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:49:38,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-20T17:49:38,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 in 173 msec 2024-11-20T17:49:38,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T17:49:38,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, REOPEN/MOVE in 502 msec 2024-11-20T17:49:38,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-20T17:49:38,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-20T17:49:38,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 930 msec 2024-11-20T17:49:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-20T17:49:38,234 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:38,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c07883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,237 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240cebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,238 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea9ca7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,241 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bfdec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,244 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efb292f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,247 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a8c4c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6d53c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,250 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29761b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,252 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ead3713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41325ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,254 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665c56d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,255 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae879b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a06cc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,262 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dbca08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,265 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c9c932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,270 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582d624d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,272 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cbbcee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,286 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10dd455c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ecd4bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf0e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480c00de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,292 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ccaf1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3f81f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:49:38,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:49:38,295 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:49:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:49:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:49:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a78fda8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:49:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:49:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:38,297 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:49:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1639e9f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:49:38,300 DEBUG [Thread-1665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,308 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,308 DEBUG [Thread-1667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,308 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,309 DEBUG [Thread-1669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,312 DEBUG [Thread-1671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,312 DEBUG [Thread-1678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,313 DEBUG [Thread-1676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:38,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:49:38,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,321 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:38,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-20T17:49:38,323 DEBUG [Thread-1682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T17:49:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,325 DEBUG [Thread-1684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:49:38,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,326 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:38,327 DEBUG [Thread-1678 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:49:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,328 DEBUG [Thread-1678 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,331 DEBUG [Thread-1684 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:49:38,332 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:38,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:38,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:49:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:49:38,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f6654f96692346a6962e76ae742f54c8_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124978317/Put/seqid=0 2024-11-20T17:49:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742191_1367 (size=12154) 2024-11-20T17:49:38,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125038387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125038388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125038390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125038393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125038391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T17:49:38,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:38,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:38,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125038497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125038507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125038507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125038507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125038517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T17:49:38,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:38,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:38,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125038707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125038717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125038718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125038718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125038727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:38,782 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:38,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f6654f96692346a6962e76ae742f54c8_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f6654f96692346a6962e76ae742f54c8_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:38,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/ae0c584f934c4c8f94f989ff33dcb520, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:38,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/ae0c584f934c4c8f94f989ff33dcb520 is 175, key is test_row_0/A:col10/1732124978317/Put/seqid=0 2024-11-20T17:49:38,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742192_1368 (size=30955) 2024-11-20T17:49:38,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/ae0c584f934c4c8f94f989ff33dcb520 2024-11-20T17:49:38,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/60b2e7520d2145fd852cc0d4719e79c1 is 50, key is test_row_0/B:col10/1732124978317/Put/seqid=0 2024-11-20T17:49:38,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742193_1369 (size=12001) 2024-11-20T17:49:38,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:38,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T17:49:39,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125039017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125039027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125039027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125039029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125039038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:39,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:39,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:39,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:39,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:39,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:39,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/60b2e7520d2145fd852cc0d4719e79c1 2024-11-20T17:49:39,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:39,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:39,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:39,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/6f851ad95507459ebf730cfceb8b0744 is 50, key is test_row_0/C:col10/1732124978317/Put/seqid=0 2024-11-20T17:49:39,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742194_1370 (size=12001) 2024-11-20T17:49:39,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/6f851ad95507459ebf730cfceb8b0744 2024-11-20T17:49:39,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/ae0c584f934c4c8f94f989ff33dcb520 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520 2024-11-20T17:49:39,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520, entries=150, sequenceid=16, filesize=30.2 K 2024-11-20T17:49:39,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/60b2e7520d2145fd852cc0d4719e79c1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/60b2e7520d2145fd852cc0d4719e79c1 2024-11-20T17:49:39,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/60b2e7520d2145fd852cc0d4719e79c1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T17:49:39,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/6f851ad95507459ebf730cfceb8b0744 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/6f851ad95507459ebf730cfceb8b0744 2024-11-20T17:49:39,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/6f851ad95507459ebf730cfceb8b0744, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T17:49:39,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d16d3ef4696b9dba5a878aefd3c902e4 in 1021ms, sequenceid=16, compaction requested=false 2024-11-20T17:49:39,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:39,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T17:49:39,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:39,409 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:49:39,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112076cfef5ca6ed40499104fd0bafa67b9a_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124978391/Put/seqid=0 2024-11-20T17:49:39,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742195_1371 (size=12154) 2024-11-20T17:49:39,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:39,457 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112076cfef5ca6ed40499104fd0bafa67b9a_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112076cfef5ca6ed40499104fd0bafa67b9a_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T17:49:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/87d405222e2641d0958b5b77aa037e0d, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/87d405222e2641d0958b5b77aa037e0d is 175, key is test_row_0/A:col10/1732124978391/Put/seqid=0 2024-11-20T17:49:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742196_1372 (size=30955) 2024-11-20T17:49:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:39,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:39,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125039560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125039565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125039567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125039569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125039570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125039677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125039677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125039687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125039687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125039687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,866 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/87d405222e2641d0958b5b77aa037e0d 2024-11-20T17:49:39,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/dce0e53694e645678ae0307d4d8c7511 is 50, key is test_row_0/B:col10/1732124978391/Put/seqid=0 2024-11-20T17:49:39,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742197_1373 (size=12001) 2024-11-20T17:49:39,877 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/dce0e53694e645678ae0307d4d8c7511 2024-11-20T17:49:39,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/7ec270e2109142edaf6023ad2857e4f5 is 50, key is test_row_0/C:col10/1732124978391/Put/seqid=0 2024-11-20T17:49:39,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742198_1374 (size=12001) 2024-11-20T17:49:39,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125039887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,892 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/7ec270e2109142edaf6023ad2857e4f5 2024-11-20T17:49:39,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125039888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/87d405222e2641d0958b5b77aa037e0d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d 2024-11-20T17:49:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125039897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125039898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,901 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-20T17:49:39,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/dce0e53694e645678ae0307d4d8c7511 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/dce0e53694e645678ae0307d4d8c7511 2024-11-20T17:49:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125039898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:39,905 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/dce0e53694e645678ae0307d4d8c7511, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T17:49:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/7ec270e2109142edaf6023ad2857e4f5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/7ec270e2109142edaf6023ad2857e4f5 2024-11-20T17:49:39,909 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/7ec270e2109142edaf6023ad2857e4f5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T17:49:39,910 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d16d3ef4696b9dba5a878aefd3c902e4 in 501ms, sequenceid=40, compaction requested=false 2024-11-20T17:49:39,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:39,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:39,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-20T17:49:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-20T17:49:39,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T17:49:39,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-20T17:49:39,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.5940 sec 2024-11-20T17:49:39,962 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:49:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:40,201 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:40,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:49:40,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d83c50e6b2334919981e47d1aa897594_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:40,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742199_1375 (size=14594) 2024-11-20T17:49:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125040242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125040243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125040243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125040244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125040245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125040347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125040357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125040357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125040358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125040358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T17:49:40,468 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:40,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-20T17:49:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T17:49:40,470 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:40,470 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:40,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125040557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125040567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125040567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125040567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125040568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T17:49:40,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:40,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d83c50e6b2334919981e47d1aa897594_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d83c50e6b2334919981e47d1aa897594_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:40,615 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/a12b8fd765a946bb90d22e87e146d72c, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:40,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/a12b8fd765a946bb90d22e87e146d72c is 175, key is test_row_0/A:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:40,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T17:49:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:40,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742200_1376 (size=39549) 2024-11-20T17:49:40,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/a12b8fd765a946bb90d22e87e146d72c 2024-11-20T17:49:40,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/1e6d1b6f98cb456bac7ab4a3dd629711 is 50, key is test_row_0/B:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:40,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742201_1377 (size=12001) 2024-11-20T17:49:40,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/1e6d1b6f98cb456bac7ab4a3dd629711 2024-11-20T17:49:40,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/bcf6042a76bb49d4be6b9e5089426196 is 50, key is test_row_0/C:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:40,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742202_1378 (size=12001) 2024-11-20T17:49:40,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T17:49:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:40,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T17:49:40,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125040867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125040877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125040886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125040887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125040887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:40,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T17:49:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:40,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:41,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/bcf6042a76bb49d4be6b9e5089426196 2024-11-20T17:49:41,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/a12b8fd765a946bb90d22e87e146d72c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c 2024-11-20T17:49:41,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c, entries=200, sequenceid=53, filesize=38.6 K 2024-11-20T17:49:41,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/1e6d1b6f98cb456bac7ab4a3dd629711 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1e6d1b6f98cb456bac7ab4a3dd629711 2024-11-20T17:49:41,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1e6d1b6f98cb456bac7ab4a3dd629711, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T17:49:41,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/bcf6042a76bb49d4be6b9e5089426196 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/bcf6042a76bb49d4be6b9e5089426196 2024-11-20T17:49:41,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T17:49:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:41,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/bcf6042a76bb49d4be6b9e5089426196, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T17:49:41,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d16d3ef4696b9dba5a878aefd3c902e4 in 885ms, sequenceid=53, compaction requested=true 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:41,086 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:41,086 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:41,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:41,087 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:41,087 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:41,087 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,087 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=99.1 K 2024-11-20T17:49:41,087 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,087 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c] 2024-11-20T17:49:41,087 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:41,087 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:41,087 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0c584f934c4c8f94f989ff33dcb520, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732124978314 2024-11-20T17:49:41,087 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,088 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/60b2e7520d2145fd852cc0d4719e79c1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/dce0e53694e645678ae0307d4d8c7511, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1e6d1b6f98cb456bac7ab4a3dd629711] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=35.2 K 2024-11-20T17:49:41,088 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d405222e2641d0958b5b77aa037e0d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732124978385 2024-11-20T17:49:41,088 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b2e7520d2145fd852cc0d4719e79c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732124978314 2024-11-20T17:49:41,088 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a12b8fd765a946bb90d22e87e146d72c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732124979553 2024-11-20T17:49:41,088 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting dce0e53694e645678ae0307d4d8c7511, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732124978385 2024-11-20T17:49:41,088 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6d1b6f98cb456bac7ab4a3dd629711, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732124979553 2024-11-20T17:49:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T17:49:41,106 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:41,107 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/122aa439318c48e59eb8be525f2cdd8f is 50, key is test_row_0/B:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:41,107 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:41,112 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112009e3cb4b7b084915b47ffff093a84e7d_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:41,114 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112009e3cb4b7b084915b47ffff093a84e7d_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:41,114 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112009e3cb4b7b084915b47ffff093a84e7d_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:41,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742203_1379 (size=12104) 2024-11-20T17:49:41,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742204_1380 (size=4469) 2024-11-20T17:49:41,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T17:49:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,234 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:49:41,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112066a7f5aea1a04347ac90060ae7c86e4c_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124980243/Put/seqid=0 2024-11-20T17:49:41,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742205_1381 (size=12154) 2024-11-20T17:49:41,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:41,293 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112066a7f5aea1a04347ac90060ae7c86e4c_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112066a7f5aea1a04347ac90060ae7c86e4c_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:41,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/3529aa3a6041491d87ecd8c80ffbce2d, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:41,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/3529aa3a6041491d87ecd8c80ffbce2d is 175, key is test_row_0/A:col10/1732124980243/Put/seqid=0 2024-11-20T17:49:41,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742206_1382 (size=30955) 2024-11-20T17:49:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:41,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:41,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125041418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125041419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125041422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125041420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125041429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,541 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/122aa439318c48e59eb8be525f2cdd8f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/122aa439318c48e59eb8be525f2cdd8f 2024-11-20T17:49:41,542 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:41,543 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d2a30f1fe5df4ba3a6ed531a04decc18 is 175, key is test_row_0/A:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125041537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125041537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125041537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125041537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125041538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742207_1383 (size=31058) 2024-11-20T17:49:41,549 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into 122aa439318c48e59eb8be525f2cdd8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:41,549 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:41,549 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=13, startTime=1732124981086; duration=0sec 2024-11-20T17:49:41,549 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:41,549 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:41,549 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:41,550 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:41,551 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:41,551 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:41,551 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/6f851ad95507459ebf730cfceb8b0744, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/7ec270e2109142edaf6023ad2857e4f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/bcf6042a76bb49d4be6b9e5089426196] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=35.2 K 2024-11-20T17:49:41,553 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f851ad95507459ebf730cfceb8b0744, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732124978314 2024-11-20T17:49:41,553 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec270e2109142edaf6023ad2857e4f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732124978385 2024-11-20T17:49:41,554 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf6042a76bb49d4be6b9e5089426196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732124979553 2024-11-20T17:49:41,557 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d2a30f1fe5df4ba3a6ed531a04decc18 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d2a30f1fe5df4ba3a6ed531a04decc18 2024-11-20T17:49:41,561 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:41,561 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/700773c1420b4f64b3c5704e0fffe7f0 is 50, key is test_row_0/C:col10/1732124980200/Put/seqid=0 2024-11-20T17:49:41,563 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into d2a30f1fe5df4ba3a6ed531a04decc18(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:41,563 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:41,563 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=13, startTime=1732124981086; duration=0sec 2024-11-20T17:49:41,563 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:41,563 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:41,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742208_1384 (size=12104) 2024-11-20T17:49:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T17:49:41,740 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/3529aa3a6041491d87ecd8c80ffbce2d 2024-11-20T17:49:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/91566d6444c0434396774feb44c1da01 is 50, key is test_row_0/B:col10/1732124980243/Put/seqid=0 2024-11-20T17:49:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125041748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125041748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125041748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125041748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742209_1385 (size=12001) 2024-11-20T17:49:41,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125041757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:41,995 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/700773c1420b4f64b3c5704e0fffe7f0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/700773c1420b4f64b3c5704e0fffe7f0 2024-11-20T17:49:42,001 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into 700773c1420b4f64b3c5704e0fffe7f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:42,001 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:42,001 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=13, startTime=1732124981086; duration=0sec 2024-11-20T17:49:42,001 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:42,001 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:42,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125042058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125042058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125042067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125042067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125042077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,159 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/91566d6444c0434396774feb44c1da01 2024-11-20T17:49:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c91fffb2616042d9b927b4a2084563d9 is 50, key is test_row_0/C:col10/1732124980243/Put/seqid=0 2024-11-20T17:49:42,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742210_1386 (size=12001) 2024-11-20T17:49:42,193 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c91fffb2616042d9b927b4a2084563d9 2024-11-20T17:49:42,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/3529aa3a6041491d87ecd8c80ffbce2d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d 2024-11-20T17:49:42,214 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d, entries=150, sequenceid=76, filesize=30.2 K 2024-11-20T17:49:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/91566d6444c0434396774feb44c1da01 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/91566d6444c0434396774feb44c1da01 2024-11-20T17:49:42,229 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/91566d6444c0434396774feb44c1da01, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T17:49:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c91fffb2616042d9b927b4a2084563d9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c91fffb2616042d9b927b4a2084563d9 2024-11-20T17:49:42,240 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c91fffb2616042d9b927b4a2084563d9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T17:49:42,241 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d16d3ef4696b9dba5a878aefd3c902e4 in 1008ms, sequenceid=76, compaction requested=false 2024-11-20T17:49:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-20T17:49:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-20T17:49:42,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T17:49:42,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-20T17:49:42,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7870 sec 2024-11-20T17:49:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:42,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:42,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T17:49:42,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207384c92a43824e95ad8d50487e375c69_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742211_1387 (size=12154) 2024-11-20T17:49:42,613 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:42,618 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207384c92a43824e95ad8d50487e375c69_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207384c92a43824e95ad8d50487e375c69_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:42,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7970d37e562e47e6aab5f3b75abc681e, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:42,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7970d37e562e47e6aab5f3b75abc681e is 175, key is test_row_0/A:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T17:49:42,628 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:42,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-20T17:49:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:42,632 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:42,632 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:42,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742212_1388 (size=30955) 2024-11-20T17:49:42,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7970d37e562e47e6aab5f3b75abc681e 2024-11-20T17:49:42,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/39192f78c17e4d98a59952ad2663aa45 is 50, key is test_row_0/B:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125042637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125042640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125042650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125042652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125042657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742213_1389 (size=12001) 2024-11-20T17:49:42,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/39192f78c17e4d98a59952ad2663aa45 2024-11-20T17:49:42,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2f13ea0023f04ea2a844d9c9e2ba742c is 50, key is test_row_0/C:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:42,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742214_1390 (size=12001) 2024-11-20T17:49:42,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2f13ea0023f04ea2a844d9c9e2ba742c 2024-11-20T17:49:42,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7970d37e562e47e6aab5f3b75abc681e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e 2024-11-20T17:49:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:42,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e, entries=150, sequenceid=94, filesize=30.2 K 2024-11-20T17:49:42,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/39192f78c17e4d98a59952ad2663aa45 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/39192f78c17e4d98a59952ad2663aa45 2024-11-20T17:49:42,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/39192f78c17e4d98a59952ad2663aa45, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T17:49:42,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2f13ea0023f04ea2a844d9c9e2ba742c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2f13ea0023f04ea2a844d9c9e2ba742c 2024-11-20T17:49:42,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2f13ea0023f04ea2a844d9c9e2ba742c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T17:49:42,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d16d3ef4696b9dba5a878aefd3c902e4 in 183ms, sequenceid=94, compaction requested=true 2024-11-20T17:49:42,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:42,760 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:42,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:42,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:42,763 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,763 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d2a30f1fe5df4ba3a6ed531a04decc18, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=90.8 K 2024-11-20T17:49:42,763 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d2a30f1fe5df4ba3a6ed531a04decc18, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e] 2024-11-20T17:49:42,764 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a30f1fe5df4ba3a6ed531a04decc18, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732124979553 2024-11-20T17:49:42,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:42,764 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3529aa3a6041491d87ecd8c80ffbce2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732124980242 2024-11-20T17:49:42,764 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7970d37e562e47e6aab5f3b75abc681e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732124981419 2024-11-20T17:49:42,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:42,768 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:42,769 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:42,769 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:42,770 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,770 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/122aa439318c48e59eb8be525f2cdd8f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/91566d6444c0434396774feb44c1da01, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/39192f78c17e4d98a59952ad2663aa45] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=35.3 K 2024-11-20T17:49:42,771 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 122aa439318c48e59eb8be525f2cdd8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732124979553 2024-11-20T17:49:42,772 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 91566d6444c0434396774feb44c1da01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732124980242 2024-11-20T17:49:42,772 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 39192f78c17e4d98a59952ad2663aa45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732124981419 2024-11-20T17:49:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:42,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:42,782 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:42,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:49:42,785 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:42,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:42,794 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:42,795 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/5c552c54c0fb4498bc40c1f8a2d5d010 is 50, key is test_row_0/B:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:42,817 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120742c32c23c6e4b7f8ba16b662bae5f1f_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:42,820 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120742c32c23c6e4b7f8ba16b662bae5f1f_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:42,820 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120742c32c23c6e4b7f8ba16b662bae5f1f_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:42,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742215_1391 (size=12207) 2024-11-20T17:49:42,839 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/5c552c54c0fb4498bc40c1f8a2d5d010 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/5c552c54c0fb4498bc40c1f8a2d5d010 2024-11-20T17:49:42,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125042822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125042824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b7847b192e8142f889d5930c0cab2223_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124982779/Put/seqid=0 2024-11-20T17:49:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,849 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into 5c552c54c0fb4498bc40c1f8a2d5d010(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:42,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:42,849 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=13, startTime=1732124982768; duration=0sec 2024-11-20T17:49:42,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:42,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:42,849 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:42,850 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:42,850 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:42,850 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,851 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/700773c1420b4f64b3c5704e0fffe7f0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c91fffb2616042d9b927b4a2084563d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2f13ea0023f04ea2a844d9c9e2ba742c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=35.3 K 2024-11-20T17:49:42,851 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 700773c1420b4f64b3c5704e0fffe7f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732124979553 2024-11-20T17:49:42,851 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c91fffb2616042d9b927b4a2084563d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732124980242 2024-11-20T17:49:42,852 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f13ea0023f04ea2a844d9c9e2ba742c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732124981419 2024-11-20T17:49:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125042841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125042843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125042843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742216_1392 (size=4469) 2024-11-20T17:49:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742217_1393 (size=17034) 2024-11-20T17:49:42,898 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:42,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/72579371a68a45f08f193076f7bbf19a is 50, key is test_row_0/C:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:42,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742218_1394 (size=12207) 2024-11-20T17:49:42,938 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/72579371a68a45f08f193076f7bbf19a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/72579371a68a45f08f193076f7bbf19a 2024-11-20T17:49:42,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:42,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:42,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:42,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:42,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:42,944 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into 72579371a68a45f08f193076f7bbf19a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:42,944 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:42,945 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=13, startTime=1732124982778; duration=0sec 2024-11-20T17:49:42,945 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:42,945 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125042948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125042948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125042967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125042967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125042967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:43,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125043170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125043177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125043187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125043188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125043188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:43,285 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#332 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:43,286 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/fcc98f91c87f483b9df6278cec6768d6 is 175, key is test_row_0/A:col10/1732124982572/Put/seqid=0 2024-11-20T17:49:43,286 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:43,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b7847b192e8142f889d5930c0cab2223_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b7847b192e8142f889d5930c0cab2223_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:43,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742219_1395 (size=31161) 2024-11-20T17:49:43,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4b01931a45bc4b4e89d1df5ed2415885, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:43,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4b01931a45bc4b4e89d1df5ed2415885 is 175, key is test_row_0/A:col10/1732124982779/Put/seqid=0 2024-11-20T17:49:43,301 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/fcc98f91c87f483b9df6278cec6768d6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fcc98f91c87f483b9df6278cec6768d6 2024-11-20T17:49:43,306 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into fcc98f91c87f483b9df6278cec6768d6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:43,306 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:43,306 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=13, startTime=1732124982760; duration=0sec 2024-11-20T17:49:43,306 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:43,306 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:43,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742220_1396 (size=48139) 2024-11-20T17:49:43,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125043477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125043497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125043509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125043509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125043509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:43,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:43,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:43,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,713 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4b01931a45bc4b4e89d1df5ed2415885 2024-11-20T17:49:43,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/a3e144a7555c4b54a8d110a381444c1a is 50, key is test_row_0/B:col10/1732124982779/Put/seqid=0 2024-11-20T17:49:43,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742221_1397 (size=12001) 2024-11-20T17:49:43,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/a3e144a7555c4b54a8d110a381444c1a 2024-11-20T17:49:43,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/4d50bb63050344cfba888a25f4e29cb3 is 50, key is test_row_0/C:col10/1732124982779/Put/seqid=0 2024-11-20T17:49:43,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742222_1398 (size=12001) 2024-11-20T17:49:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:43,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:43,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:43,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:43,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:43,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125043997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:44,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:44,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:44,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:44,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125044017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125044027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125044027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125044027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:44,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/4d50bb63050344cfba888a25f4e29cb3 2024-11-20T17:49:44,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:44,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4b01931a45bc4b4e89d1df5ed2415885 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885 2024-11-20T17:49:44,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885, entries=250, sequenceid=117, filesize=47.0 K 2024-11-20T17:49:44,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/a3e144a7555c4b54a8d110a381444c1a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a3e144a7555c4b54a8d110a381444c1a 2024-11-20T17:49:44,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a3e144a7555c4b54a8d110a381444c1a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T17:49:44,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/4d50bb63050344cfba888a25f4e29cb3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d50bb63050344cfba888a25f4e29cb3 2024-11-20T17:49:44,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d50bb63050344cfba888a25f4e29cb3, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T17:49:44,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d16d3ef4696b9dba5a878aefd3c902e4 in 1404ms, sequenceid=117, compaction requested=false 2024-11-20T17:49:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:44,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T17:49:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:44,320 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T17:49:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203e78547a3a5249b0a1b35e1059ed09d6_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124982839/Put/seqid=0 2024-11-20T17:49:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742223_1399 (size=12254) 2024-11-20T17:49:44,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:44,733 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203e78547a3a5249b0a1b35e1059ed09d6_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203e78547a3a5249b0a1b35e1059ed09d6_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/543793a2ca2e4ed5bcb423621b83108f, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/543793a2ca2e4ed5bcb423621b83108f is 175, key is test_row_0/A:col10/1732124982839/Put/seqid=0 2024-11-20T17:49:44,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742224_1400 (size=31055) 2024-11-20T17:49:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:45,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:45,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125045075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125045076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125045076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125045076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125045087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,152 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/543793a2ca2e4ed5bcb423621b83108f 2024-11-20T17:49:45,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/f0c1b785ca3d491a940e530a98978936 is 50, key is test_row_0/B:col10/1732124982839/Put/seqid=0 2024-11-20T17:49:45,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742225_1401 (size=12101) 2024-11-20T17:49:45,164 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/f0c1b785ca3d491a940e530a98978936 2024-11-20T17:49:45,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/db1cb7a41cb64d6a9435fdb6bede656d is 50, key is test_row_0/C:col10/1732124982839/Put/seqid=0 2024-11-20T17:49:45,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742226_1402 (size=12101) 2024-11-20T17:49:45,173 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/db1cb7a41cb64d6a9435fdb6bede656d 2024-11-20T17:49:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/543793a2ca2e4ed5bcb423621b83108f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f 2024-11-20T17:49:45,179 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f, entries=150, sequenceid=134, filesize=30.3 K 2024-11-20T17:49:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/f0c1b785ca3d491a940e530a98978936 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/f0c1b785ca3d491a940e530a98978936 2024-11-20T17:49:45,182 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/f0c1b785ca3d491a940e530a98978936, entries=150, sequenceid=134, filesize=11.8 K 2024-11-20T17:49:45,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/db1cb7a41cb64d6a9435fdb6bede656d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/db1cb7a41cb64d6a9435fdb6bede656d 2024-11-20T17:49:45,186 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/db1cb7a41cb64d6a9435fdb6bede656d, entries=150, sequenceid=134, filesize=11.8 K 2024-11-20T17:49:45,186 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d16d3ef4696b9dba5a878aefd3c902e4 in 866ms, sequenceid=134, compaction requested=true 2024-11-20T17:49:45,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:45,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:45,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-20T17:49:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-20T17:49:45,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T17:49:45,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5540 sec 2024-11-20T17:49:45,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.5600 sec 2024-11-20T17:49:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:45,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:45,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:49:45,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208ba3961649dd4263a5608f087baecd36_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:45,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742227_1403 (size=14794) 2024-11-20T17:49:45,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:45,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208ba3961649dd4263a5608f087baecd36_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208ba3961649dd4263a5608f087baecd36_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:45,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/21e061d10f1448079060d9f6e9465f3f, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:45,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/21e061d10f1448079060d9f6e9465f3f is 175, key is test_row_0/A:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:45,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742228_1404 (size=39749) 2024-11-20T17:49:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125045220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125045221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125045223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125045233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125045234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125045337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125045337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125045338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125045338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125045347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125045548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125045548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125045548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125045548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125045557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/21e061d10f1448079060d9f6e9465f3f 2024-11-20T17:49:45,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/a82473e3c3284d8cb5fc0ab3d97a5bc9 is 50, key is test_row_0/B:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:45,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742229_1405 (size=12151) 2024-11-20T17:49:45,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/a82473e3c3284d8cb5fc0ab3d97a5bc9 2024-11-20T17:49:45,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/3c9d77c497e44410b33828c98f32e398 is 50, key is test_row_0/C:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:45,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742230_1406 (size=12151) 2024-11-20T17:49:45,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125045867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125045867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125045867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125045867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125045867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/3c9d77c497e44410b33828c98f32e398 2024-11-20T17:49:46,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/21e061d10f1448079060d9f6e9465f3f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f 2024-11-20T17:49:46,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f, entries=200, sequenceid=157, filesize=38.8 K 2024-11-20T17:49:46,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/a82473e3c3284d8cb5fc0ab3d97a5bc9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a82473e3c3284d8cb5fc0ab3d97a5bc9 2024-11-20T17:49:46,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a82473e3c3284d8cb5fc0ab3d97a5bc9, entries=150, sequenceid=157, filesize=11.9 K 2024-11-20T17:49:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/3c9d77c497e44410b33828c98f32e398 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/3c9d77c497e44410b33828c98f32e398 2024-11-20T17:49:46,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/3c9d77c497e44410b33828c98f32e398, entries=150, sequenceid=157, filesize=11.9 K 2024-11-20T17:49:46,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d16d3ef4696b9dba5a878aefd3c902e4 in 894ms, sequenceid=157, compaction requested=true 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:46,099 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:49:46,099 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:46,101 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:46,101 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:46,102 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:46,102 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fcc98f91c87f483b9df6278cec6768d6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=146.6 K 2024-11-20T17:49:46,102 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:46,102 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fcc98f91c87f483b9df6278cec6768d6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f] 2024-11-20T17:49:46,102 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:46,102 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:46,102 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:46,102 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/5c552c54c0fb4498bc40c1f8a2d5d010, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a3e144a7555c4b54a8d110a381444c1a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/f0c1b785ca3d491a940e530a98978936, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a82473e3c3284d8cb5fc0ab3d97a5bc9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=47.3 K 2024-11-20T17:49:46,102 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcc98f91c87f483b9df6278cec6768d6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732124981419 2024-11-20T17:49:46,103 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c552c54c0fb4498bc40c1f8a2d5d010, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732124981419 2024-11-20T17:49:46,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b01931a45bc4b4e89d1df5ed2415885, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732124982644 2024-11-20T17:49:46,103 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a3e144a7555c4b54a8d110a381444c1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732124982649 2024-11-20T17:49:46,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543793a2ca2e4ed5bcb423621b83108f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732124982839 2024-11-20T17:49:46,103 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c1b785ca3d491a940e530a98978936, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732124982839 2024-11-20T17:49:46,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e061d10f1448079060d9f6e9465f3f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124985067 2024-11-20T17:49:46,104 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a82473e3c3284d8cb5fc0ab3d97a5bc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124985072 2024-11-20T17:49:46,117 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:46,132 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#345 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:46,132 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/c5901ed9c85e46b2ba792cf70f71afb4 is 50, key is test_row_0/B:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:46,145 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120d46d9783bdcb4c338f5cf0e858cf18f3_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:46,147 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120d46d9783bdcb4c338f5cf0e858cf18f3_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:46,147 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d46d9783bdcb4c338f5cf0e858cf18f3_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:46,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742231_1407 (size=12493) 2024-11-20T17:49:46,190 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/c5901ed9c85e46b2ba792cf70f71afb4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/c5901ed9c85e46b2ba792cf70f71afb4 2024-11-20T17:49:46,206 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into c5901ed9c85e46b2ba792cf70f71afb4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:46,206 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:46,206 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=12, startTime=1732124986099; duration=0sec 2024-11-20T17:49:46,207 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:46,208 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:46,208 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:46,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742232_1408 (size=4469) 2024-11-20T17:49:46,211 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:46,211 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:46,211 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:46,211 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/72579371a68a45f08f193076f7bbf19a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d50bb63050344cfba888a25f4e29cb3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/db1cb7a41cb64d6a9435fdb6bede656d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/3c9d77c497e44410b33828c98f32e398] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=47.3 K 2024-11-20T17:49:46,212 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 72579371a68a45f08f193076f7bbf19a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732124981419 2024-11-20T17:49:46,212 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d50bb63050344cfba888a25f4e29cb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732124982649 2024-11-20T17:49:46,212 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting db1cb7a41cb64d6a9435fdb6bede656d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732124982839 2024-11-20T17:49:46,213 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9d77c497e44410b33828c98f32e398, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124985072 2024-11-20T17:49:46,228 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:46,228 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/218198bc368144059e1063b6debf3de9 is 50, key is test_row_0/C:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742233_1409 (size=12493) 2024-11-20T17:49:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:46,391 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:46,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T17:49:46,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120721c9ec5abb64851b46f307337a84e66_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124985220/Put/seqid=0 2024-11-20T17:49:46,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742234_1410 (size=14794) 2024-11-20T17:49:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125046429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125046430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125046430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125046431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125046433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125046537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125046547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125046547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125046547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125046547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,611 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#344 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:46,612 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/a687fe10c89f41a38d838ec1c4922d20 is 175, key is test_row_0/A:col10/1732124985203/Put/seqid=0 2024-11-20T17:49:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742235_1411 (size=31447) 2024-11-20T17:49:46,624 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/a687fe10c89f41a38d838ec1c4922d20 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a687fe10c89f41a38d838ec1c4922d20 2024-11-20T17:49:46,630 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into a687fe10c89f41a38d838ec1c4922d20(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:46,630 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:46,630 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=12, startTime=1732124986099; duration=0sec 2024-11-20T17:49:46,630 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:46,630 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:46,656 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/218198bc368144059e1063b6debf3de9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/218198bc368144059e1063b6debf3de9 2024-11-20T17:49:46,665 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into 218198bc368144059e1063b6debf3de9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:46,665 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:46,665 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=12, startTime=1732124986099; duration=0sec 2024-11-20T17:49:46,666 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:46,666 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:46,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125046747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125046757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125046757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125046758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125046767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T17:49:46,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:46,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-20T17:49:46,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:46,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:46,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:46,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:46,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:46,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120721c9ec5abb64851b46f307337a84e66_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120721c9ec5abb64851b46f307337a84e66_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:46,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/036e42000ad7408ea54197c8bbad4919, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:46,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/036e42000ad7408ea54197c8bbad4919 is 175, key is test_row_0/A:col10/1732124985220/Put/seqid=0 2024-11-20T17:49:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742236_1412 (size=39749) 2024-11-20T17:49:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:46,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:46,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:46,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125047067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125047077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125047077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125047077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125047077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:47,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:47,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:47,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/036e42000ad7408ea54197c8bbad4919 2024-11-20T17:49:47,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/0bd986bcacc949858d979b660f74387d is 50, key is test_row_0/B:col10/1732124985220/Put/seqid=0 2024-11-20T17:49:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742237_1413 (size=12151) 2024-11-20T17:49:47,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:47,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125047577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125047587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125047588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125047588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125047588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:47,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/0bd986bcacc949858d979b660f74387d 2024-11-20T17:49:47,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2e9b7897b6614941947699e5f9d1e1ac is 50, key is test_row_0/C:col10/1732124985220/Put/seqid=0 2024-11-20T17:49:47,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742238_1414 (size=12151) 2024-11-20T17:49:47,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:47,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:48,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:48,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:48,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:48,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2e9b7897b6614941947699e5f9d1e1ac 2024-11-20T17:49:48,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/036e42000ad7408ea54197c8bbad4919 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919 2024-11-20T17:49:48,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919, entries=200, sequenceid=173, filesize=38.8 K 2024-11-20T17:49:48,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/0bd986bcacc949858d979b660f74387d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0bd986bcacc949858d979b660f74387d 2024-11-20T17:49:48,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0bd986bcacc949858d979b660f74387d, entries=150, sequenceid=173, filesize=11.9 K 2024-11-20T17:49:48,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2e9b7897b6614941947699e5f9d1e1ac as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2e9b7897b6614941947699e5f9d1e1ac 2024-11-20T17:49:48,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2e9b7897b6614941947699e5f9d1e1ac, entries=150, sequenceid=173, filesize=11.9 K 2024-11-20T17:49:48,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d16d3ef4696b9dba5a878aefd3c902e4 in 1716ms, sequenceid=173, compaction requested=false 2024-11-20T17:49:48,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:48,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T17:49:48,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:48,187 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T17:49:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4c461a44c5447aba8f9c7d51fc67f60_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124986428/Put/seqid=0 2024-11-20T17:49:48,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742239_1415 (size=12304) 2024-11-20T17:49:48,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:48,609 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4c461a44c5447aba8f9c7d51fc67f60_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4c461a44c5447aba8f9c7d51fc67f60_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/9c7960c537ec4827a4c8bc5440f26acd, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/9c7960c537ec4827a4c8bc5440f26acd is 175, key is test_row_0/A:col10/1732124986428/Put/seqid=0 2024-11-20T17:49:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742240_1416 (size=31105) 2024-11-20T17:49:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125048621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125048619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125048622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125048633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125048640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125048747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125048747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125048748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125048757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125048757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:48,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125048957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125048967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125048967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125048967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:48,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125048977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,015 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/9c7960c537ec4827a4c8bc5440f26acd 2024-11-20T17:49:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/957be366f92044cc987e732c1a24620d is 50, key is test_row_0/B:col10/1732124986428/Put/seqid=0 2024-11-20T17:49:49,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742241_1417 (size=12151) 2024-11-20T17:49:49,034 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/957be366f92044cc987e732c1a24620d 2024-11-20T17:49:49,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/88b6654cf341484aa8bfb11a4fe75b9a is 50, key is test_row_0/C:col10/1732124986428/Put/seqid=0 2024-11-20T17:49:49,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742242_1418 (size=12151) 2024-11-20T17:49:49,086 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/88b6654cf341484aa8bfb11a4fe75b9a 2024-11-20T17:49:49,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/9c7960c537ec4827a4c8bc5440f26acd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd 2024-11-20T17:49:49,097 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd, entries=150, sequenceid=196, filesize=30.4 K 2024-11-20T17:49:49,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/957be366f92044cc987e732c1a24620d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/957be366f92044cc987e732c1a24620d 2024-11-20T17:49:49,102 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/957be366f92044cc987e732c1a24620d, entries=150, sequenceid=196, filesize=11.9 K 2024-11-20T17:49:49,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/88b6654cf341484aa8bfb11a4fe75b9a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/88b6654cf341484aa8bfb11a4fe75b9a 2024-11-20T17:49:49,114 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/88b6654cf341484aa8bfb11a4fe75b9a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-20T17:49:49,115 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d16d3ef4696b9dba5a878aefd3c902e4 in 928ms, sequenceid=196, compaction requested=true 2024-11-20T17:49:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-20T17:49:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-20T17:49:49,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T17:49:49,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3160 sec 2024-11-20T17:49:49,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.3210 sec 2024-11-20T17:49:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:49,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T17:49:49,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d9efaf56bd384a47803de28ae0a9efcf_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742243_1419 (size=14794) 2024-11-20T17:49:49,308 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:49,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d9efaf56bd384a47803de28ae0a9efcf_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d9efaf56bd384a47803de28ae0a9efcf_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:49,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4dd8702b876a4640bd0c6b1b2b68d9ca, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:49,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4dd8702b876a4640bd0c6b1b2b68d9ca is 175, key is test_row_0/A:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742244_1420 (size=39749) 2024-11-20T17:49:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125049321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125049323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125049324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125049324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125049333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125049437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125049447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125049447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125049447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125049448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125049650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125049667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125049667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125049667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125049667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:49,719 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4dd8702b876a4640bd0c6b1b2b68d9ca 2024-11-20T17:49:49,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/639946aec154401a85d4253dc750a76a is 50, key is test_row_0/B:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742245_1421 (size=12151) 2024-11-20T17:49:49,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/639946aec154401a85d4253dc750a76a 2024-11-20T17:49:49,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/13470916dc194148aa3ef1619873faeb is 50, key is test_row_0/C:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742246_1422 (size=12151) 2024-11-20T17:49:49,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/13470916dc194148aa3ef1619873faeb 2024-11-20T17:49:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/4dd8702b876a4640bd0c6b1b2b68d9ca as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca 2024-11-20T17:49:49,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca, entries=200, sequenceid=212, filesize=38.8 K 2024-11-20T17:49:49,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/639946aec154401a85d4253dc750a76a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/639946aec154401a85d4253dc750a76a 2024-11-20T17:49:49,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/639946aec154401a85d4253dc750a76a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T17:49:49,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/13470916dc194148aa3ef1619873faeb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/13470916dc194148aa3ef1619873faeb 2024-11-20T17:49:49,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/13470916dc194148aa3ef1619873faeb, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T17:49:49,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d16d3ef4696b9dba5a878aefd3c902e4 in 604ms, sequenceid=212, compaction requested=true 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:49,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T17:49:49,884 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:49,885 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:49,886 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:49,886 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:49,886 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/218198bc368144059e1063b6debf3de9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2e9b7897b6614941947699e5f9d1e1ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/88b6654cf341484aa8bfb11a4fe75b9a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/13470916dc194148aa3ef1619873faeb] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=47.8 K 2024-11-20T17:49:49,886 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a687fe10c89f41a38d838ec1c4922d20, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=138.7 K 2024-11-20T17:49:49,886 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a687fe10c89f41a38d838ec1c4922d20, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca] 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 218198bc368144059e1063b6debf3de9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124985072 2024-11-20T17:49:49,886 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a687fe10c89f41a38d838ec1c4922d20, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124985072 2024-11-20T17:49:49,887 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e9b7897b6614941947699e5f9d1e1ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732124985206 2024-11-20T17:49:49,887 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036e42000ad7408ea54197c8bbad4919, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732124985206 2024-11-20T17:49:49,887 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b6654cf341484aa8bfb11a4fe75b9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732124986428 2024-11-20T17:49:49,887 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7960c537ec4827a4c8bc5440f26acd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732124986428 2024-11-20T17:49:49,887 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 13470916dc194148aa3ef1619873faeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732124988630 2024-11-20T17:49:49,887 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dd8702b876a4640bd0c6b1b2b68d9ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732124988615 2024-11-20T17:49:49,897 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:49,897 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/5a754c31521a45149f5831623e098e4e is 50, key is test_row_0/C:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,898 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:49,902 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209f39d17e80bb47be9cc87e9628772ba0_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:49,905 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209f39d17e80bb47be9cc87e9628772ba0_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:49,905 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209f39d17e80bb47be9cc87e9628772ba0_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:49,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742247_1423 (size=12629) 2024-11-20T17:49:49,922 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/5a754c31521a45149f5831623e098e4e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/5a754c31521a45149f5831623e098e4e 2024-11-20T17:49:49,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742248_1424 (size=4469) 2024-11-20T17:49:49,924 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#357 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:49,925 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7b5382228f1a4a588e38b06ab79174ac is 175, key is test_row_0/A:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,927 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into 5a754c31521a45149f5831623e098e4e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:49,927 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:49,927 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=12, startTime=1732124989884; duration=0sec 2024-11-20T17:49:49,927 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:49,927 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:49,928 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:49,930 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:49,930 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:49,930 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:49,930 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/c5901ed9c85e46b2ba792cf70f71afb4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0bd986bcacc949858d979b660f74387d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/957be366f92044cc987e732c1a24620d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/639946aec154401a85d4253dc750a76a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=47.8 K 2024-11-20T17:49:49,931 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c5901ed9c85e46b2ba792cf70f71afb4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732124985072 2024-11-20T17:49:49,931 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd986bcacc949858d979b660f74387d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732124985206 2024-11-20T17:49:49,932 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 957be366f92044cc987e732c1a24620d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732124986428 2024-11-20T17:49:49,933 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 639946aec154401a85d4253dc750a76a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732124988630 2024-11-20T17:49:49,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742249_1425 (size=31583) 2024-11-20T17:49:49,943 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:49,944 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/1a42766f01c4476fa24d21e724023c47 is 50, key is test_row_0/B:col10/1732124989279/Put/seqid=0 2024-11-20T17:49:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:49,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:49,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T17:49:49,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201573d6959bd14fa699a311775847ed0b_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124989315/Put/seqid=0 2024-11-20T17:49:49,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742250_1426 (size=12629) 2024-11-20T17:49:50,010 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/1a42766f01c4476fa24d21e724023c47 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1a42766f01c4476fa24d21e724023c47 2024-11-20T17:49:50,015 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into 1a42766f01c4476fa24d21e724023c47(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:50,015 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:50,015 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=12, startTime=1732124989884; duration=0sec 2024-11-20T17:49:50,015 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:50,015 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:50,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742251_1427 (size=14794) 2024-11-20T17:49:50,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125050021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125050022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125050023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125050023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125050024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125050147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125050147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125050147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125050147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125050148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,342 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7b5382228f1a4a588e38b06ab79174ac as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7b5382228f1a4a588e38b06ab79174ac 2024-11-20T17:49:50,347 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into 7b5382228f1a4a588e38b06ab79174ac(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:50,347 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:50,347 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=12, startTime=1732124989884; duration=0sec 2024-11-20T17:49:50,347 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:50,347 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:50,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125050357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125050357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125050357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125050368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125050368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,430 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:50,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201573d6959bd14fa699a311775847ed0b_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201573d6959bd14fa699a311775847ed0b_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:50,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/6b8c16b1afdd42338b5079896f594b5d, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:50,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/6b8c16b1afdd42338b5079896f594b5d is 175, key is test_row_0/A:col10/1732124989315/Put/seqid=0 2024-11-20T17:49:50,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742252_1428 (size=39749) 2024-11-20T17:49:50,461 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/6b8c16b1afdd42338b5079896f594b5d 2024-11-20T17:49:50,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/9debcb41b50f4489a3fcc54cf9d21cb8 is 50, key is test_row_0/B:col10/1732124989315/Put/seqid=0 2024-11-20T17:49:50,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742253_1429 (size=12151) 2024-11-20T17:49:50,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/9debcb41b50f4489a3fcc54cf9d21cb8 2024-11-20T17:49:50,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/a829fa83a74d42daaf8e1bd7049e4133 is 50, key is test_row_0/C:col10/1732124989315/Put/seqid=0 2024-11-20T17:49:50,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742254_1430 (size=12151) 2024-11-20T17:49:50,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/a829fa83a74d42daaf8e1bd7049e4133 2024-11-20T17:49:50,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/6b8c16b1afdd42338b5079896f594b5d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d 2024-11-20T17:49:50,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d, entries=200, sequenceid=234, filesize=38.8 K 2024-11-20T17:49:50,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/9debcb41b50f4489a3fcc54cf9d21cb8 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/9debcb41b50f4489a3fcc54cf9d21cb8 2024-11-20T17:49:50,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/9debcb41b50f4489a3fcc54cf9d21cb8, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T17:49:50,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/a829fa83a74d42daaf8e1bd7049e4133 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/a829fa83a74d42daaf8e1bd7049e4133 2024-11-20T17:49:50,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/a829fa83a74d42daaf8e1bd7049e4133, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T17:49:50,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d16d3ef4696b9dba5a878aefd3c902e4 in 604ms, sequenceid=234, compaction requested=false 2024-11-20T17:49:50,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:50,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:50,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T17:49:50,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202f5c51118dac48c7ae96592ca2ec630d_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:50,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742255_1431 (size=12304) 2024-11-20T17:49:50,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:50,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202f5c51118dac48c7ae96592ca2ec630d_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202f5c51118dac48c7ae96592ca2ec630d_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:50,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/fba1092547dc402096c305c198bea061, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:50,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/fba1092547dc402096c305c198bea061 is 175, key is test_row_0/A:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:50,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742256_1432 (size=31105) 2024-11-20T17:49:50,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/fba1092547dc402096c305c198bea061 2024-11-20T17:49:50,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125050736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125050736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125050737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125050738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/da5eced3e6d8467685d8bf7e4b71c3fd is 50, key is test_row_0/B:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125050742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742257_1433 (size=12151) 2024-11-20T17:49:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125050857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125050857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125050858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125050858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125050867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T17:49:50,977 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:50,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-20T17:49:50,979 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T17:49:50,980 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:50,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:51,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125051067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125051068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125051068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125051068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T17:49:51,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125051087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T17:49:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:51,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/da5eced3e6d8467685d8bf7e4b71c3fd 2024-11-20T17:49:51,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:49:51,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/4d781e51a8584fd9a39f43b9ec8a9844 is 50, key is test_row_0/C:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:51,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742258_1434 (size=12151) 2024-11-20T17:49:51,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/4d781e51a8584fd9a39f43b9ec8a9844 2024-11-20T17:49:51,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/fba1092547dc402096c305c198bea061 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061 2024-11-20T17:49:51,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061, entries=150, sequenceid=252, filesize=30.4 K 2024-11-20T17:49:51,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/da5eced3e6d8467685d8bf7e4b71c3fd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/da5eced3e6d8467685d8bf7e4b71c3fd 2024-11-20T17:49:51,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/da5eced3e6d8467685d8bf7e4b71c3fd, entries=150, sequenceid=252, filesize=11.9 K 2024-11-20T17:49:51,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/4d781e51a8584fd9a39f43b9ec8a9844 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d781e51a8584fd9a39f43b9ec8a9844 2024-11-20T17:49:51,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d781e51a8584fd9a39f43b9ec8a9844, entries=150, sequenceid=252, filesize=11.9 K 2024-11-20T17:49:51,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d16d3ef4696b9dba5a878aefd3c902e4 in 564ms, sequenceid=252, compaction requested=true 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:51,250 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:51,250 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:51,251 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:51,251 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:51,251 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,251 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1a42766f01c4476fa24d21e724023c47, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/9debcb41b50f4489a3fcc54cf9d21cb8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/da5eced3e6d8467685d8bf7e4b71c3fd] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=36.1 K 2024-11-20T17:49:51,251 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:51,251 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:51,252 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,252 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7b5382228f1a4a588e38b06ab79174ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=100.0 K 2024-11-20T17:49:51,252 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,252 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7b5382228f1a4a588e38b06ab79174ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061] 2024-11-20T17:49:51,252 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a42766f01c4476fa24d21e724023c47, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732124988630 2024-11-20T17:49:51,252 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5382228f1a4a588e38b06ab79174ac, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732124988630 2024-11-20T17:49:51,252 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9debcb41b50f4489a3fcc54cf9d21cb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732124989315 2024-11-20T17:49:51,253 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8c16b1afdd42338b5079896f594b5d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732124989315 2024-11-20T17:49:51,253 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting da5eced3e6d8467685d8bf7e4b71c3fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124990020 2024-11-20T17:49:51,253 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba1092547dc402096c305c198bea061, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124990020 2024-11-20T17:49:51,283 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:51,283 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/37f7014f0ab7473ebbbc0df8bca936b1 is 50, key is test_row_0/B:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:51,290 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:51,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T17:49:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,290 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T17:49:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T17:49:51,313 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112084f5027aac274d6ba00e2d22f61874f5_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:51,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742259_1435 (size=12731) 2024-11-20T17:49:51,315 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112084f5027aac274d6ba00e2d22f61874f5_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:51,315 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112084f5027aac274d6ba00e2d22f61874f5_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:51,320 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/37f7014f0ab7473ebbbc0df8bca936b1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/37f7014f0ab7473ebbbc0df8bca936b1 2024-11-20T17:49:51,326 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into 37f7014f0ab7473ebbbc0df8bca936b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:51,326 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:51,326 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=13, startTime=1732124991250; duration=0sec 2024-11-20T17:49:51,327 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:51,327 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:51,327 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:51,328 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:51,328 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:51,328 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:51,328 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/5a754c31521a45149f5831623e098e4e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/a829fa83a74d42daaf8e1bd7049e4133, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d781e51a8584fd9a39f43b9ec8a9844] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=36.1 K 2024-11-20T17:49:51,329 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a754c31521a45149f5831623e098e4e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732124988630 2024-11-20T17:49:51,330 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a829fa83a74d42daaf8e1bd7049e4133, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732124989315 2024-11-20T17:49:51,330 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d781e51a8584fd9a39f43b9ec8a9844, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124990020 2024-11-20T17:49:51,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205020a99f59a4438c85ade923c0516b1f_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124990737/Put/seqid=0 2024-11-20T17:49:51,348 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:51,348 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/422fe45d4bd54d60a9ea68bbb049f3bc is 50, key is test_row_0/C:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:51,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742260_1436 (size=4469) 2024-11-20T17:49:51,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742261_1437 (size=12454) 2024-11-20T17:49:51,374 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#366 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:51,374 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/9ac35b4e8e594feb98c155c1aeb55ee1 is 175, key is test_row_0/A:col10/1732124990022/Put/seqid=0 2024-11-20T17:49:51,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742262_1438 (size=12731) 2024-11-20T17:49:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:51,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742263_1439 (size=31685) 2024-11-20T17:49:51,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125051459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125051460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125051461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125051462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125051464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125051567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125051577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125051588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125051588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125051588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T17:49:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205020a99f59a4438c85ade923c0516b1f_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205020a99f59a4438c85ade923c0516b1f_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/f4912132adf34ee1ac6f8f6d1270de37, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/f4912132adf34ee1ac6f8f6d1270de37 is 175, key is test_row_0/A:col10/1732124990737/Put/seqid=0 2024-11-20T17:49:51,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742264_1440 (size=31255) 2024-11-20T17:49:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125051787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125051788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,797 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/422fe45d4bd54d60a9ea68bbb049f3bc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/422fe45d4bd54d60a9ea68bbb049f3bc 2024-11-20T17:49:51,804 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into 422fe45d4bd54d60a9ea68bbb049f3bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:51,804 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:51,804 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=13, startTime=1732124991250; duration=0sec 2024-11-20T17:49:51,804 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:51,804 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:51,805 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/9ac35b4e8e594feb98c155c1aeb55ee1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9ac35b4e8e594feb98c155c1aeb55ee1 2024-11-20T17:49:51,811 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into 9ac35b4e8e594feb98c155c1aeb55ee1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:51,811 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:51,811 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=13, startTime=1732124991250; duration=0sec 2024-11-20T17:49:51,811 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:51,811 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:51,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125051807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125051807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125051807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125052097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125052097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T17:49:52,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125052127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125052127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125052128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,183 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/f4912132adf34ee1ac6f8f6d1270de37 2024-11-20T17:49:52,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/ba9a100438c54e2fb164ea7d4ff0b2ec is 50, key is test_row_0/B:col10/1732124990737/Put/seqid=0 2024-11-20T17:49:52,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742265_1441 (size=12301) 2024-11-20T17:49:52,208 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/ba9a100438c54e2fb164ea7d4ff0b2ec 2024-11-20T17:49:52,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/02c9b886d7be4d7da02b26c35c3515d4 is 50, key is test_row_0/C:col10/1732124990737/Put/seqid=0 2024-11-20T17:49:52,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742266_1442 (size=12301) 2024-11-20T17:49:52,239 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/02c9b886d7be4d7da02b26c35c3515d4 2024-11-20T17:49:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/f4912132adf34ee1ac6f8f6d1270de37 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37 2024-11-20T17:49:52,254 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37, entries=150, sequenceid=272, filesize=30.5 K 2024-11-20T17:49:52,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/ba9a100438c54e2fb164ea7d4ff0b2ec as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/ba9a100438c54e2fb164ea7d4ff0b2ec 2024-11-20T17:49:52,261 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/ba9a100438c54e2fb164ea7d4ff0b2ec, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T17:49:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/02c9b886d7be4d7da02b26c35c3515d4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/02c9b886d7be4d7da02b26c35c3515d4 2024-11-20T17:49:52,268 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/02c9b886d7be4d7da02b26c35c3515d4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T17:49:52,269 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d16d3ef4696b9dba5a878aefd3c902e4 in 978ms, sequenceid=272, compaction requested=false 2024-11-20T17:49:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-20T17:49:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-20T17:49:52,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T17:49:52,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-11-20T17:49:52,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.2970 sec 2024-11-20T17:49:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:52,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T17:49:52,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201cfb53c9c6a74dd5833c12296bfd4cbc_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:52,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742267_1443 (size=14994) 2024-11-20T17:49:52,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:52,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201cfb53c9c6a74dd5833c12296bfd4cbc_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201cfb53c9c6a74dd5833c12296bfd4cbc_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:52,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/057298be6105432b978bac22964478d7, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:52,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/057298be6105432b978bac22964478d7 is 175, key is test_row_0/A:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:52,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742268_1444 (size=39949) 2024-11-20T17:49:52,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125052663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125052663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125052664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125052664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125052665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125052787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125052787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125052787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125052787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125052787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125053008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125053008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125053008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125053008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125053008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/057298be6105432b978bac22964478d7 2024-11-20T17:49:53,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/0768dad1cd6f4206bce6f5bf28f04782 is 50, key is test_row_0/B:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:53,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742269_1445 (size=12301) 2024-11-20T17:49:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T17:49:53,128 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:53,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-20T17:49:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T17:49:53,131 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:53,132 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:53,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T17:49:53,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T17:49:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125053327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125053328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125053328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125053328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125053339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T17:49:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T17:49:53,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/0768dad1cd6f4206bce6f5bf28f04782 2024-11-20T17:49:53,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c1a128cabcfb44d59adf771b8392731c is 50, key is test_row_0/C:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:53,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742270_1446 (size=12301) 2024-11-20T17:49:53,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c1a128cabcfb44d59adf771b8392731c 2024-11-20T17:49:53,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/057298be6105432b978bac22964478d7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7 2024-11-20T17:49:53,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7, entries=200, sequenceid=294, filesize=39.0 K 2024-11-20T17:49:53,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/0768dad1cd6f4206bce6f5bf28f04782 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0768dad1cd6f4206bce6f5bf28f04782 2024-11-20T17:49:53,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0768dad1cd6f4206bce6f5bf28f04782, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:49:53,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c1a128cabcfb44d59adf771b8392731c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c1a128cabcfb44d59adf771b8392731c 2024-11-20T17:49:53,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c1a128cabcfb44d59adf771b8392731c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-20T17:49:53,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d16d3ef4696b9dba5a878aefd3c902e4 in 965ms, sequenceid=294, compaction requested=true 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:53,580 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:53,580 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:53,583 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:53,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:53,583 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:53,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:53,583 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,583 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,583 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9ac35b4e8e594feb98c155c1aeb55ee1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=100.5 K 2024-11-20T17:49:53,583 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/37f7014f0ab7473ebbbc0df8bca936b1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/ba9a100438c54e2fb164ea7d4ff0b2ec, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0768dad1cd6f4206bce6f5bf28f04782] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=36.5 K 2024-11-20T17:49:53,583 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,583 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9ac35b4e8e594feb98c155c1aeb55ee1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7] 2024-11-20T17:49:53,584 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ac35b4e8e594feb98c155c1aeb55ee1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124990020 2024-11-20T17:49:53,584 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f7014f0ab7473ebbbc0df8bca936b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124990020 2024-11-20T17:49:53,584 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4912132adf34ee1ac6f8f6d1270de37, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732124990735 2024-11-20T17:49:53,584 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057298be6105432b978bac22964478d7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124991458 2024-11-20T17:49:53,584 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9a100438c54e2fb164ea7d4ff0b2ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732124990735 2024-11-20T17:49:53,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0768dad1cd6f4206bce6f5bf28f04782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124991458 2024-11-20T17:49:53,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T17:49:53,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,597 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T17:49:53,604 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:53,608 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:53,609 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/d074b157884a46c68cbb802903b8bfba is 50, key is test_row_0/B:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:53,613 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b934f5b970d144b7b512e2498af7f8ee_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:53,616 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b934f5b970d144b7b512e2498af7f8ee_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:53,616 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b934f5b970d144b7b512e2498af7f8ee_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201a79a42d82c54836a63246244abb2c79_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124992661/Put/seqid=0 2024-11-20T17:49:53,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742271_1447 (size=12983) 2024-11-20T17:49:53,660 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/d074b157884a46c68cbb802903b8bfba as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/d074b157884a46c68cbb802903b8bfba 2024-11-20T17:49:53,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742272_1448 (size=4469) 2024-11-20T17:49:53,684 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#374 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:53,685 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d6688531c8bf4dfa86aff2cac7122d49 is 175, key is test_row_0/A:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:53,699 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into d074b157884a46c68cbb802903b8bfba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:53,699 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:53,699 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=13, startTime=1732124993580; duration=0sec 2024-11-20T17:49:53,699 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:53,699 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:53,699 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:53,700 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:53,700 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:53,700 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,700 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/422fe45d4bd54d60a9ea68bbb049f3bc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/02c9b886d7be4d7da02b26c35c3515d4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c1a128cabcfb44d59adf771b8392731c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=36.5 K 2024-11-20T17:49:53,701 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 422fe45d4bd54d60a9ea68bbb049f3bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732124990020 2024-11-20T17:49:53,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742274_1450 (size=31937) 2024-11-20T17:49:53,713 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c9b886d7be4d7da02b26c35c3515d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732124990735 2024-11-20T17:49:53,713 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a128cabcfb44d59adf771b8392731c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124991458 2024-11-20T17:49:53,719 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d6688531c8bf4dfa86aff2cac7122d49 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d6688531c8bf4dfa86aff2cac7122d49 2024-11-20T17:49:53,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742273_1449 (size=12454) 2024-11-20T17:49:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,730 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into d6688531c8bf4dfa86aff2cac7122d49(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:53,730 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:53,730 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=13, startTime=1732124993580; duration=0sec 2024-11-20T17:49:53,731 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:53,731 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:53,734 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#377 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:53,734 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/49aa448621134b5e98b3693cdba52ce9 is 50, key is test_row_0/C:col10/1732124992614/Put/seqid=0 2024-11-20T17:49:53,739 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201a79a42d82c54836a63246244abb2c79_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201a79a42d82c54836a63246244abb2c79_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d0ef2d8a615a4e9abc8aef0a16faf68b, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:53,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d0ef2d8a615a4e9abc8aef0a16faf68b is 175, key is test_row_0/A:col10/1732124992661/Put/seqid=0 2024-11-20T17:49:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T17:49:53,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742275_1451 (size=12983) 2024-11-20T17:49:53,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742276_1452 (size=31255) 2024-11-20T17:49:53,770 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d0ef2d8a615a4e9abc8aef0a16faf68b 2024-11-20T17:49:53,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/adeee6bedafa47f68efe25ed6f784d65 is 50, key is test_row_0/B:col10/1732124992661/Put/seqid=0 2024-11-20T17:49:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742277_1453 (size=12301) 2024-11-20T17:49:53,822 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/adeee6bedafa47f68efe25ed6f784d65 2024-11-20T17:49:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2632b14572744f4f9fc06abcf1e5ae06 is 50, key is test_row_0/C:col10/1732124992661/Put/seqid=0 2024-11-20T17:49:53,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742278_1454 (size=12301) 2024-11-20T17:49:53,845 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2632b14572744f4f9fc06abcf1e5ae06 2024-11-20T17:49:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d0ef2d8a615a4e9abc8aef0a16faf68b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b 2024-11-20T17:49:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:53,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:53,858 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b, entries=150, sequenceid=311, filesize=30.5 K 2024-11-20T17:49:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/adeee6bedafa47f68efe25ed6f784d65 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adeee6bedafa47f68efe25ed6f784d65 2024-11-20T17:49:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,864 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adeee6bedafa47f68efe25ed6f784d65, entries=150, sequenceid=311, filesize=12.0 K 2024-11-20T17:49:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2632b14572744f4f9fc06abcf1e5ae06 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2632b14572744f4f9fc06abcf1e5ae06 2024-11-20T17:49:53,869 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2632b14572744f4f9fc06abcf1e5ae06, entries=150, sequenceid=311, filesize=12.0 K 2024-11-20T17:49:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,870 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=46.96 KB/48090 for d16d3ef4696b9dba5a878aefd3c902e4 in 273ms, sequenceid=311, compaction requested=false 2024-11-20T17:49:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-20T17:49:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-20T17:49:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:53,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:49:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T17:49:53,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 740 msec 2024-11-20T17:49:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 746 msec 2024-11-20T17:49:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f2e03eeedaa5477f8930fa7e65a24701_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124993856/Put/seqid=0 2024-11-20T17:49:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742280_1456 (size=27698) 2024-11-20T17:49:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,902 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f2e03eeedaa5477f8930fa7e65a24701_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f2e03eeedaa5477f8930fa7e65a24701_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,903 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/45e6160ac42d450a84f2d7299dd711e6, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/45e6160ac42d450a84f2d7299dd711e6 is 175, key is test_row_0/A:col10/1732124993856/Put/seqid=0 2024-11-20T17:49:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742279_1455 (size=83485) 2024-11-20T17:49:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/45e6160ac42d450a84f2d7299dd711e6 2024-11-20T17:49:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:53,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/adf9e1a33a5844b3bfd494d3d325f0da is 50, key is test_row_0/B:col10/1732124993856/Put/seqid=0 2024-11-20T17:49:53,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742281_1457 (size=26945) 2024-11-20T17:49:53,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/adf9e1a33a5844b3bfd494d3d325f0da 2024-11-20T17:49:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125053957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125053957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125053960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125053963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125053964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:53,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/61e711a58a214ac38b3587d11a9992f3 is 50, key is test_row_0/C:col10/1732124993856/Put/seqid=0 2024-11-20T17:49:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742282_1458 (size=12301) 2024-11-20T17:49:54,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/61e711a58a214ac38b3587d11a9992f3 2024-11-20T17:49:54,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/45e6160ac42d450a84f2d7299dd711e6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6 2024-11-20T17:49:54,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6, entries=450, sequenceid=326, filesize=81.5 K 2024-11-20T17:49:54,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/adf9e1a33a5844b3bfd494d3d325f0da as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adf9e1a33a5844b3bfd494d3d325f0da 2024-11-20T17:49:54,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adf9e1a33a5844b3bfd494d3d325f0da, entries=450, sequenceid=326, filesize=26.3 K 2024-11-20T17:49:54,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/61e711a58a214ac38b3587d11a9992f3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/61e711a58a214ac38b3587d11a9992f3 2024-11-20T17:49:54,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/61e711a58a214ac38b3587d11a9992f3, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T17:49:54,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d16d3ef4696b9dba5a878aefd3c902e4 in 153ms, sequenceid=326, compaction requested=true 2024-11-20T17:49:54,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:49:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T17:49:54,026 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-20T17:49:54,027 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:49:54,027 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:49:54,027 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. because compaction request was cancelled 2024-11-20T17:49:54,027 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:54,027 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:54,027 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 146677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:54,028 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:54,028 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,028 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d6688531c8bf4dfa86aff2cac7122d49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=143.2 K 2024-11-20T17:49:54,028 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,028 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d6688531c8bf4dfa86aff2cac7122d49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6] 2024-11-20T17:49:54,028 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6688531c8bf4dfa86aff2cac7122d49, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124991458 2024-11-20T17:49:54,031 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ef2d8a615a4e9abc8aef0a16faf68b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732124992661 2024-11-20T17:49:54,031 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e6160ac42d450a84f2d7299dd711e6, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732124993851 2024-11-20T17:49:54,053 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:54,085 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411200eae32e739654347b67338e86793df45_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:54,089 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411200eae32e739654347b67338e86793df45_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:54,089 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200eae32e739654347b67338e86793df45_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:54,103 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:54,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:54,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742283_1459 (size=4469) 2024-11-20T17:49:54,135 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#383 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:54,137 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/514b93136fd34c2bb8c85b3b56bf59d9 is 175, key is test_row_0/A:col10/1732124993856/Put/seqid=0 2024-11-20T17:49:54,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206b7c2ac1a11c49c9b6910ad441dc31a1_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124994101/Put/seqid=0 2024-11-20T17:49:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742284_1460 (size=32039) 2024-11-20T17:49:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125054117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125054124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742285_1461 (size=14994) 2024-11-20T17:49:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125054136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125054140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125054141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,170 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/49aa448621134b5e98b3693cdba52ce9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/49aa448621134b5e98b3693cdba52ce9 2024-11-20T17:49:54,176 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into 49aa448621134b5e98b3693cdba52ce9(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:54,176 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:54,176 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=13, startTime=1732124993580; duration=0sec 2024-11-20T17:49:54,177 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:54,177 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:54,177 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:54,178 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 52229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:54,178 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:54,178 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,178 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/d074b157884a46c68cbb802903b8bfba, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adeee6bedafa47f68efe25ed6f784d65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adf9e1a33a5844b3bfd494d3d325f0da] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=51.0 K 2024-11-20T17:49:54,178 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d074b157884a46c68cbb802903b8bfba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124991458 2024-11-20T17:49:54,179 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting adeee6bedafa47f68efe25ed6f784d65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732124992661 2024-11-20T17:49:54,179 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting adf9e1a33a5844b3bfd494d3d325f0da, keycount=450, bloomtype=ROW, size=26.3 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732124993851 2024-11-20T17:49:54,194 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:54,194 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/8349f3c2666849a5aac073404408fadc is 50, key is test_row_0/B:col10/1732124993856/Put/seqid=0 2024-11-20T17:49:54,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742286_1462 (size=13085) 2024-11-20T17:49:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125054247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125054247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125054257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T17:49:54,268 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:54,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:49:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-20T17:49:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:54,271 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:49:54,271 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:49:54,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:49:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125054268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125054267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:54,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125054457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125054458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125054477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125054487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125054487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,544 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/514b93136fd34c2bb8c85b3b56bf59d9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/514b93136fd34c2bb8c85b3b56bf59d9 2024-11-20T17:49:54,548 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into 514b93136fd34c2bb8c85b3b56bf59d9(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:54,548 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:54,548 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=13, startTime=1732124994026; duration=0sec 2024-11-20T17:49:54,548 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:54,548 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:54,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:54,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206b7c2ac1a11c49c9b6910ad441dc31a1_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206b7c2ac1a11c49c9b6910ad441dc31a1_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:54,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7c2f4ccf799a4114b045657769c72645, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:54,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7c2f4ccf799a4114b045657769c72645 is 175, key is test_row_0/A:col10/1732124994101/Put/seqid=0 2024-11-20T17:49:54,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742287_1463 (size=39949) 2024-11-20T17:49:54,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:54,613 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/8349f3c2666849a5aac073404408fadc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8349f3c2666849a5aac073404408fadc 2024-11-20T17:49:54,620 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into 8349f3c2666849a5aac073404408fadc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:54,621 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:54,621 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=13, startTime=1732124994026; duration=0sec 2024-11-20T17:49:54,621 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:54,621 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:54,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125054777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125054777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125054787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125054807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125054807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:54,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:54,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:54,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7c2f4ccf799a4114b045657769c72645 2024-11-20T17:49:54,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/05f7a86c24544feca5d9b1d411ef67f9 is 50, key is test_row_0/B:col10/1732124994101/Put/seqid=0 2024-11-20T17:49:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742288_1464 (size=12301) 2024-11-20T17:49:55,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125055287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:55,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125055297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125055297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:55,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125055317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:55,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125055327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:55,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/05f7a86c24544feca5d9b1d411ef67f9 2024-11-20T17:49:55,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/d432673997a6426d8e07c1f0ba48864f is 50, key is test_row_0/C:col10/1732124994101/Put/seqid=0 2024-11-20T17:49:55,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742289_1465 (size=12301) 2024-11-20T17:49:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:55,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,637 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d2a30f1fe5df4ba3a6ed531a04decc18, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fcc98f91c87f483b9df6278cec6768d6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a687fe10c89f41a38d838ec1c4922d20, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7b5382228f1a4a588e38b06ab79174ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9ac35b4e8e594feb98c155c1aeb55ee1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d6688531c8bf4dfa86aff2cac7122d49, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6] to archive 2024-11-20T17:49:55,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:55,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/ae0c584f934c4c8f94f989ff33dcb520 2024-11-20T17:49:55,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/87d405222e2641d0958b5b77aa037e0d 2024-11-20T17:49:55,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a12b8fd765a946bb90d22e87e146d72c 2024-11-20T17:49:55,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d2a30f1fe5df4ba3a6ed531a04decc18 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d2a30f1fe5df4ba3a6ed531a04decc18 2024-11-20T17:49:55,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/3529aa3a6041491d87ecd8c80ffbce2d 2024-11-20T17:49:55,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fcc98f91c87f483b9df6278cec6768d6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fcc98f91c87f483b9df6278cec6768d6 2024-11-20T17:49:55,651 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7970d37e562e47e6aab5f3b75abc681e 2024-11-20T17:49:55,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4b01931a45bc4b4e89d1df5ed2415885 2024-11-20T17:49:55,654 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/543793a2ca2e4ed5bcb423621b83108f 2024-11-20T17:49:55,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/21e061d10f1448079060d9f6e9465f3f 2024-11-20T17:49:55,658 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a687fe10c89f41a38d838ec1c4922d20 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/a687fe10c89f41a38d838ec1c4922d20 2024-11-20T17:49:55,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/036e42000ad7408ea54197c8bbad4919 2024-11-20T17:49:55,660 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9c7960c537ec4827a4c8bc5440f26acd 2024-11-20T17:49:55,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/4dd8702b876a4640bd0c6b1b2b68d9ca 2024-11-20T17:49:55,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7b5382228f1a4a588e38b06ab79174ac to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7b5382228f1a4a588e38b06ab79174ac 2024-11-20T17:49:55,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/6b8c16b1afdd42338b5079896f594b5d 2024-11-20T17:49:55,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9ac35b4e8e594feb98c155c1aeb55ee1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/9ac35b4e8e594feb98c155c1aeb55ee1 2024-11-20T17:49:55,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/fba1092547dc402096c305c198bea061 2024-11-20T17:49:55,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/f4912132adf34ee1ac6f8f6d1270de37 2024-11-20T17:49:55,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/057298be6105432b978bac22964478d7 2024-11-20T17:49:55,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d6688531c8bf4dfa86aff2cac7122d49 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d6688531c8bf4dfa86aff2cac7122d49 2024-11-20T17:49:55,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d0ef2d8a615a4e9abc8aef0a16faf68b 2024-11-20T17:49:55,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/45e6160ac42d450a84f2d7299dd711e6 2024-11-20T17:49:55,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/60b2e7520d2145fd852cc0d4719e79c1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/dce0e53694e645678ae0307d4d8c7511, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/122aa439318c48e59eb8be525f2cdd8f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1e6d1b6f98cb456bac7ab4a3dd629711, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/91566d6444c0434396774feb44c1da01, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/5c552c54c0fb4498bc40c1f8a2d5d010, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/39192f78c17e4d98a59952ad2663aa45, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a3e144a7555c4b54a8d110a381444c1a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/f0c1b785ca3d491a940e530a98978936, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/c5901ed9c85e46b2ba792cf70f71afb4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a82473e3c3284d8cb5fc0ab3d97a5bc9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0bd986bcacc949858d979b660f74387d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/957be366f92044cc987e732c1a24620d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1a42766f01c4476fa24d21e724023c47, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/639946aec154401a85d4253dc750a76a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/9debcb41b50f4489a3fcc54cf9d21cb8, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/37f7014f0ab7473ebbbc0df8bca936b1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/da5eced3e6d8467685d8bf7e4b71c3fd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/ba9a100438c54e2fb164ea7d4ff0b2ec, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/d074b157884a46c68cbb802903b8bfba, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0768dad1cd6f4206bce6f5bf28f04782, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adeee6bedafa47f68efe25ed6f784d65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adf9e1a33a5844b3bfd494d3d325f0da] to archive 2024-11-20T17:49:55,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:55,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/60b2e7520d2145fd852cc0d4719e79c1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/60b2e7520d2145fd852cc0d4719e79c1 2024-11-20T17:49:55,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/dce0e53694e645678ae0307d4d8c7511 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/dce0e53694e645678ae0307d4d8c7511 2024-11-20T17:49:55,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/122aa439318c48e59eb8be525f2cdd8f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/122aa439318c48e59eb8be525f2cdd8f 2024-11-20T17:49:55,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1e6d1b6f98cb456bac7ab4a3dd629711 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1e6d1b6f98cb456bac7ab4a3dd629711 2024-11-20T17:49:55,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/91566d6444c0434396774feb44c1da01 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/91566d6444c0434396774feb44c1da01 2024-11-20T17:49:55,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/5c552c54c0fb4498bc40c1f8a2d5d010 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/5c552c54c0fb4498bc40c1f8a2d5d010 2024-11-20T17:49:55,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/39192f78c17e4d98a59952ad2663aa45 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/39192f78c17e4d98a59952ad2663aa45 2024-11-20T17:49:55,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a3e144a7555c4b54a8d110a381444c1a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a3e144a7555c4b54a8d110a381444c1a 2024-11-20T17:49:55,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/f0c1b785ca3d491a940e530a98978936 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/f0c1b785ca3d491a940e530a98978936 2024-11-20T17:49:55,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/c5901ed9c85e46b2ba792cf70f71afb4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/c5901ed9c85e46b2ba792cf70f71afb4 2024-11-20T17:49:55,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a82473e3c3284d8cb5fc0ab3d97a5bc9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/a82473e3c3284d8cb5fc0ab3d97a5bc9 2024-11-20T17:49:55,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0bd986bcacc949858d979b660f74387d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0bd986bcacc949858d979b660f74387d 2024-11-20T17:49:55,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/957be366f92044cc987e732c1a24620d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/957be366f92044cc987e732c1a24620d 2024-11-20T17:49:55,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1a42766f01c4476fa24d21e724023c47 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/1a42766f01c4476fa24d21e724023c47 2024-11-20T17:49:55,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/639946aec154401a85d4253dc750a76a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/639946aec154401a85d4253dc750a76a 2024-11-20T17:49:55,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/9debcb41b50f4489a3fcc54cf9d21cb8 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/9debcb41b50f4489a3fcc54cf9d21cb8 2024-11-20T17:49:55,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/37f7014f0ab7473ebbbc0df8bca936b1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/37f7014f0ab7473ebbbc0df8bca936b1 2024-11-20T17:49:55,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/da5eced3e6d8467685d8bf7e4b71c3fd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/da5eced3e6d8467685d8bf7e4b71c3fd 2024-11-20T17:49:55,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/ba9a100438c54e2fb164ea7d4ff0b2ec to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/ba9a100438c54e2fb164ea7d4ff0b2ec 2024-11-20T17:49:55,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/d074b157884a46c68cbb802903b8bfba to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/d074b157884a46c68cbb802903b8bfba 2024-11-20T17:49:55,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0768dad1cd6f4206bce6f5bf28f04782 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/0768dad1cd6f4206bce6f5bf28f04782 2024-11-20T17:49:55,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adeee6bedafa47f68efe25ed6f784d65 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adeee6bedafa47f68efe25ed6f784d65 2024-11-20T17:49:55,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adf9e1a33a5844b3bfd494d3d325f0da to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/adf9e1a33a5844b3bfd494d3d325f0da 2024-11-20T17:49:55,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/6f851ad95507459ebf730cfceb8b0744, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/7ec270e2109142edaf6023ad2857e4f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/700773c1420b4f64b3c5704e0fffe7f0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/bcf6042a76bb49d4be6b9e5089426196, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c91fffb2616042d9b927b4a2084563d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/72579371a68a45f08f193076f7bbf19a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2f13ea0023f04ea2a844d9c9e2ba742c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d50bb63050344cfba888a25f4e29cb3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/db1cb7a41cb64d6a9435fdb6bede656d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/218198bc368144059e1063b6debf3de9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/3c9d77c497e44410b33828c98f32e398, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2e9b7897b6614941947699e5f9d1e1ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/88b6654cf341484aa8bfb11a4fe75b9a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/5a754c31521a45149f5831623e098e4e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/13470916dc194148aa3ef1619873faeb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/a829fa83a74d42daaf8e1bd7049e4133, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/422fe45d4bd54d60a9ea68bbb049f3bc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d781e51a8584fd9a39f43b9ec8a9844, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/02c9b886d7be4d7da02b26c35c3515d4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c1a128cabcfb44d59adf771b8392731c] to archive 2024-11-20T17:49:55,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:49:55,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/6f851ad95507459ebf730cfceb8b0744 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/6f851ad95507459ebf730cfceb8b0744 2024-11-20T17:49:55,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/7ec270e2109142edaf6023ad2857e4f5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/7ec270e2109142edaf6023ad2857e4f5 2024-11-20T17:49:55,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/700773c1420b4f64b3c5704e0fffe7f0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/700773c1420b4f64b3c5704e0fffe7f0 2024-11-20T17:49:55,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/bcf6042a76bb49d4be6b9e5089426196 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/bcf6042a76bb49d4be6b9e5089426196 2024-11-20T17:49:55,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c91fffb2616042d9b927b4a2084563d9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c91fffb2616042d9b927b4a2084563d9 2024-11-20T17:49:55,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/72579371a68a45f08f193076f7bbf19a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/72579371a68a45f08f193076f7bbf19a 2024-11-20T17:49:55,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2f13ea0023f04ea2a844d9c9e2ba742c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2f13ea0023f04ea2a844d9c9e2ba742c 2024-11-20T17:49:55,705 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d50bb63050344cfba888a25f4e29cb3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d50bb63050344cfba888a25f4e29cb3 2024-11-20T17:49:55,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/db1cb7a41cb64d6a9435fdb6bede656d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/db1cb7a41cb64d6a9435fdb6bede656d 2024-11-20T17:49:55,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/218198bc368144059e1063b6debf3de9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/218198bc368144059e1063b6debf3de9 2024-11-20T17:49:55,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/3c9d77c497e44410b33828c98f32e398 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/3c9d77c497e44410b33828c98f32e398 2024-11-20T17:49:55,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2e9b7897b6614941947699e5f9d1e1ac to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2e9b7897b6614941947699e5f9d1e1ac 2024-11-20T17:49:55,711 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/88b6654cf341484aa8bfb11a4fe75b9a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/88b6654cf341484aa8bfb11a4fe75b9a 2024-11-20T17:49:55,711 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/5a754c31521a45149f5831623e098e4e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/5a754c31521a45149f5831623e098e4e 2024-11-20T17:49:55,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/13470916dc194148aa3ef1619873faeb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/13470916dc194148aa3ef1619873faeb 2024-11-20T17:49:55,713 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/a829fa83a74d42daaf8e1bd7049e4133 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/a829fa83a74d42daaf8e1bd7049e4133 2024-11-20T17:49:55,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/422fe45d4bd54d60a9ea68bbb049f3bc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/422fe45d4bd54d60a9ea68bbb049f3bc 2024-11-20T17:49:55,717 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d781e51a8584fd9a39f43b9ec8a9844 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/4d781e51a8584fd9a39f43b9ec8a9844 2024-11-20T17:49:55,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/02c9b886d7be4d7da02b26c35c3515d4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/02c9b886d7be4d7da02b26c35c3515d4 2024-11-20T17:49:55,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c1a128cabcfb44d59adf771b8392731c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c1a128cabcfb44d59adf771b8392731c 2024-11-20T17:49:55,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/d432673997a6426d8e07c1f0ba48864f 2024-11-20T17:49:55,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:49:55,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/7c2f4ccf799a4114b045657769c72645 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645 2024-11-20T17:49:55,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645, entries=200, sequenceid=351, filesize=39.0 K 2024-11-20T17:49:55,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/05f7a86c24544feca5d9b1d411ef67f9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/05f7a86c24544feca5d9b1d411ef67f9 2024-11-20T17:49:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/05f7a86c24544feca5d9b1d411ef67f9, entries=150, sequenceid=351, filesize=12.0 K 2024-11-20T17:49:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/d432673997a6426d8e07c1f0ba48864f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d432673997a6426d8e07c1f0ba48864f 2024-11-20T17:49:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d432673997a6426d8e07c1f0ba48864f, entries=150, sequenceid=351, filesize=12.0 K 2024-11-20T17:49:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d16d3ef4696b9dba5a878aefd3c902e4 in 1715ms, sequenceid=351, compaction requested=true 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T17:49:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:49:55,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. because compaction request was cancelled 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:55,819 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,820 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T17:49:55,820 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T17:49:55,820 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. because compaction request was cancelled 2024-11-20T17:49:55,820 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,820 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:55,820 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,821 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,821 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/49aa448621134b5e98b3693cdba52ce9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2632b14572744f4f9fc06abcf1e5ae06, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/61e711a58a214ac38b3587d11a9992f3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d432673997a6426d8e07c1f0ba48864f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=48.7 K 2024-11-20T17:49:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,821 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 49aa448621134b5e98b3693cdba52ce9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732124991458 2024-11-20T17:49:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,821 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2632b14572744f4f9fc06abcf1e5ae06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732124992661 2024-11-20T17:49:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,822 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e711a58a214ac38b3587d11a9992f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732124993856 2024-11-20T17:49:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,822 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d432673997a6426d8e07c1f0ba48864f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732124993949 2024-11-20T17:49:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,830 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,830 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/ebc285115cd6433d85f8a5ca76593df9 is 50, key is test_row_0/C:col10/1732124994101/Put/seqid=0 2024-11-20T17:49:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742290_1466 (size=12439) 2024-11-20T17:49:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T17:49:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,953 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:49:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203648e0a2370f4f5395a411ac83fe2d27_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124994133/Put/seqid=0 2024-11-20T17:49:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742291_1467 (size=9914) 2024-11-20T17:49:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,253 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/ebc285115cd6433d85f8a5ca76593df9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/ebc285115cd6433d85f8a5ca76593df9 2024-11-20T17:49:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,258 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into ebc285115cd6433d85f8a5ca76593df9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:56,258 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:56,258 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=12, startTime=1732124995819; duration=0sec 2024-11-20T17:49:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,258 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:56,258 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:56,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. as already flushing 2024-11-20T17:49:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:56,382 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203648e0a2370f4f5395a411ac83fe2d27_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203648e0a2370f4f5395a411ac83fe2d27_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:56,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/94b06347256b47efbebb5f29b2665109, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:56,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/94b06347256b47efbebb5f29b2665109 is 175, key is test_row_0/A:col10/1732124994133/Put/seqid=0 2024-11-20T17:49:56,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742292_1468 (size=22561) 2024-11-20T17:49:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125056409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125056410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125056411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125056411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125056412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125056527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125056527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125056527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125056528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125056528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125056737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125056737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125056738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125056738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125056747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:56,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:56,791 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/94b06347256b47efbebb5f29b2665109 2024-11-20T17:49:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/07ddc183a539417cbf67dfb1b72f2e86 is 50, key is test_row_0/B:col10/1732124994133/Put/seqid=0 2024-11-20T17:49:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742293_1469 (size=9857) 2024-11-20T17:49:56,801 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/07ddc183a539417cbf67dfb1b72f2e86 2024-11-20T17:49:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/d702f8ebe540417cb3846b123efb1dc9 is 50, key is test_row_0/C:col10/1732124994133/Put/seqid=0 2024-11-20T17:49:56,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742294_1470 (size=9857) 2024-11-20T17:49:56,829 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/d702f8ebe540417cb3846b123efb1dc9 2024-11-20T17:49:56,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/94b06347256b47efbebb5f29b2665109 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109 2024-11-20T17:49:56,838 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109, entries=100, sequenceid=366, filesize=22.0 K 2024-11-20T17:49:56,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/07ddc183a539417cbf67dfb1b72f2e86 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/07ddc183a539417cbf67dfb1b72f2e86 2024-11-20T17:49:56,843 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/07ddc183a539417cbf67dfb1b72f2e86, entries=100, sequenceid=366, filesize=9.6 K 2024-11-20T17:49:56,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/d702f8ebe540417cb3846b123efb1dc9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d702f8ebe540417cb3846b123efb1dc9 2024-11-20T17:49:56,850 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d702f8ebe540417cb3846b123efb1dc9, entries=100, sequenceid=366, filesize=9.6 K 2024-11-20T17:49:56,851 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d16d3ef4696b9dba5a878aefd3c902e4 in 898ms, sequenceid=366, compaction requested=true 2024-11-20T17:49:56,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:56,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:56,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-20T17:49:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-20T17:49:56,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-20T17:49:56,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5830 sec 2024-11-20T17:49:56,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.5880 sec 2024-11-20T17:49:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:57,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:49:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:57,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112050f9f70411e34f59a3240b39297b4e43_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742295_1471 (size=14994) 2024-11-20T17:49:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125057068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125057067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125057068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125057078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125057080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125057188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125057188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125057188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125057188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125057197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125057398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125057398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125057398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125057398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125057407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:57,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112050f9f70411e34f59a3240b39297b4e43_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112050f9f70411e34f59a3240b39297b4e43_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:57,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d8f7ca172d474a069a792a9b16202576, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:57,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d8f7ca172d474a069a792a9b16202576 is 175, key is test_row_0/A:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:57,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742296_1472 (size=39949) 2024-11-20T17:49:57,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d8f7ca172d474a069a792a9b16202576 2024-11-20T17:49:57,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/fe7296f82ce84204a274ffb4cf2b888b is 50, key is test_row_0/B:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:57,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742297_1473 (size=12301) 2024-11-20T17:49:57,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/fe7296f82ce84204a274ffb4cf2b888b 2024-11-20T17:49:57,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/1209a0726e0d4b4081f528d9f34f8e31 is 50, key is test_row_0/C:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:57,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742298_1474 (size=12301) 2024-11-20T17:49:57,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125057707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125057717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125057717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125057717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125057727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:57,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/1209a0726e0d4b4081f528d9f34f8e31 2024-11-20T17:49:57,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/d8f7ca172d474a069a792a9b16202576 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576 2024-11-20T17:49:57,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576, entries=200, sequenceid=392, filesize=39.0 K 2024-11-20T17:49:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/fe7296f82ce84204a274ffb4cf2b888b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/fe7296f82ce84204a274ffb4cf2b888b 2024-11-20T17:49:57,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/fe7296f82ce84204a274ffb4cf2b888b, entries=150, sequenceid=392, filesize=12.0 K 2024-11-20T17:49:57,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/1209a0726e0d4b4081f528d9f34f8e31 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/1209a0726e0d4b4081f528d9f34f8e31 2024-11-20T17:49:57,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/1209a0726e0d4b4081f528d9f34f8e31, entries=150, sequenceid=392, filesize=12.0 K 2024-11-20T17:49:57,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d16d3ef4696b9dba5a878aefd3c902e4 in 877ms, sequenceid=392, compaction requested=true 2024-11-20T17:49:57,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:57,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:49:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:57,930 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:49:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:57,930 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:49:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:49:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:57,931 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:57,931 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/A is initiating minor compaction (all files) 2024-11-20T17:49:57,931 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/A in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:57,931 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/514b93136fd34c2bb8c85b3b56bf59d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=131.3 K 2024-11-20T17:49:57,931 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:57,931 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/514b93136fd34c2bb8c85b3b56bf59d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576] 2024-11-20T17:49:57,931 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:49:57,931 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/B is initiating minor compaction (all files) 2024-11-20T17:49:57,931 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/B in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:57,931 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8349f3c2666849a5aac073404408fadc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/05f7a86c24544feca5d9b1d411ef67f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/07ddc183a539417cbf67dfb1b72f2e86, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/fe7296f82ce84204a274ffb4cf2b888b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=46.4 K 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 514b93136fd34c2bb8c85b3b56bf59d9, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732124993856 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8349f3c2666849a5aac073404408fadc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732124993856 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c2f4ccf799a4114b045657769c72645, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732124993949 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b06347256b47efbebb5f29b2665109, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732124994133 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f7a86c24544feca5d9b1d411ef67f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732124993949 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ddc183a539417cbf67dfb1b72f2e86, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732124994133 2024-11-20T17:49:57,932 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f7ca172d474a069a792a9b16202576, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732124996407 2024-11-20T17:49:57,933 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting fe7296f82ce84204a274ffb4cf2b888b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732124996409 2024-11-20T17:49:57,944 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:57,946 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#B#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:57,947 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/8ed94142c2504d2b885b09ffc2b80286 is 50, key is test_row_0/B:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:57,948 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ccd874b0975248afb0e0113329fa61d5_d16d3ef4696b9dba5a878aefd3c902e4 store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:57,950 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ccd874b0975248afb0e0113329fa61d5_d16d3ef4696b9dba5a878aefd3c902e4, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:57,951 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ccd874b0975248afb0e0113329fa61d5_d16d3ef4696b9dba5a878aefd3c902e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:57,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742299_1475 (size=12439) 2024-11-20T17:49:57,957 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/8ed94142c2504d2b885b09ffc2b80286 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8ed94142c2504d2b885b09ffc2b80286 2024-11-20T17:49:57,966 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/B of d16d3ef4696b9dba5a878aefd3c902e4 into 8ed94142c2504d2b885b09ffc2b80286(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:57,966 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:57,966 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/B, priority=12, startTime=1732124997930; duration=0sec 2024-11-20T17:49:57,967 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:49:57,967 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:49:57,967 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:49:57,967 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:49:57,967 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): d16d3ef4696b9dba5a878aefd3c902e4/C is initiating minor compaction (all files) 2024-11-20T17:49:57,967 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d16d3ef4696b9dba5a878aefd3c902e4/C in TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:49:57,968 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/ebc285115cd6433d85f8a5ca76593df9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d702f8ebe540417cb3846b123efb1dc9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/1209a0726e0d4b4081f528d9f34f8e31] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp, totalSize=33.8 K 2024-11-20T17:49:57,968 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc285115cd6433d85f8a5ca76593df9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732124993949 2024-11-20T17:49:57,968 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d702f8ebe540417cb3846b123efb1dc9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732124994133 2024-11-20T17:49:57,968 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1209a0726e0d4b4081f528d9f34f8e31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732124996409 2024-11-20T17:49:57,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742300_1476 (size=4469) 2024-11-20T17:49:57,978 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:57,979 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c634d97028054047a14541c928e3fd18 is 50, key is test_row_0/C:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:57,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742301_1477 (size=12541) 2024-11-20T17:49:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:58,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:49:58,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:49:58,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207035cd47cba942bb87ada98f0775aa31_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732124998230/Put/seqid=0 2024-11-20T17:49:58,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742302_1478 (size=14994) 2024-11-20T17:49:58,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125058258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125058267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125058269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125058269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125058270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,318 INFO [Thread-1676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1676. 2024-11-20T17:49:58,318 INFO [Thread-1684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1684. 2024-11-20T17:49:58,318 DEBUG [Thread-1676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:58,318 DEBUG [Thread-1684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:58,318 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,318 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,318 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,318 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:58,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:58,319 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-11-20T17:49:58,319 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:58,319 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,319 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:58,327 INFO [Thread-1678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1678. 2024-11-20T17:49:58,327 DEBUG [Thread-1678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:58,327 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,327 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:58,327 INFO [Thread-1682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1682. 2024-11-20T17:49:58,328 DEBUG [Thread-1682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:49:58,328 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,328 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:49:58,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:49:58,370 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d16d3ef4696b9dba5a878aefd3c902e4#A#compaction#396 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:49:58,371 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/b5255371b6674c4d90b402cb49ddb03f is 175, key is test_row_0/A:col10/1732124996409/Put/seqid=0 2024-11-20T17:49:58,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742303_1479 (size=31393) 2024-11-20T17:49:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125058377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125058377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,387 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/c634d97028054047a14541c928e3fd18 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c634d97028054047a14541c928e3fd18 2024-11-20T17:49:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125058387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125058387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125058387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,391 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/C of d16d3ef4696b9dba5a878aefd3c902e4 into c634d97028054047a14541c928e3fd18(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:58,391 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:58,391 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/C, priority=13, startTime=1732124997930; duration=0sec 2024-11-20T17:49:58,391 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:58,391 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:49:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T17:49:58,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:49:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125058587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125058587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125058598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125058598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125058598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:49:58,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207035cd47cba942bb87ada98f0775aa31_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207035cd47cba942bb87ada98f0775aa31_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:49:58,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/256fea829a1249d79b20400d2b471e66, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:49:58,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/256fea829a1249d79b20400d2b471e66 is 175, key is test_row_0/A:col10/1732124998230/Put/seqid=0 2024-11-20T17:49:58,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742304_1480 (size=39949) 2024-11-20T17:49:58,778 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/b5255371b6674c4d90b402cb49ddb03f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/b5255371b6674c4d90b402cb49ddb03f 2024-11-20T17:49:58,782 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d16d3ef4696b9dba5a878aefd3c902e4/A of d16d3ef4696b9dba5a878aefd3c902e4 into b5255371b6674c4d90b402cb49ddb03f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:49:58,782 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:49:58,782 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., storeName=d16d3ef4696b9dba5a878aefd3c902e4/A, priority=12, startTime=1732124997929; duration=0sec 2024-11-20T17:49:58,782 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:49:58,782 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:49:58,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125058897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125058897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125058907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125058907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125058907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:59,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/256fea829a1249d79b20400d2b471e66 2024-11-20T17:49:59,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/54ed2aed2de34aa4bc4d1732e86e218a is 50, key is test_row_0/B:col10/1732124998230/Put/seqid=0 2024-11-20T17:49:59,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742305_1481 (size=12301) 2024-11-20T17:49:59,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38960 deadline: 1732125059407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38940 deadline: 1732125059407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:49:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38980 deadline: 1732125059417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38976 deadline: 1732125059417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38924 deadline: 1732125059417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:49:59,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/54ed2aed2de34aa4bc4d1732e86e218a 2024-11-20T17:49:59,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/12feb2bc09cf4a529263ce4515f5e810 is 50, key is test_row_0/C:col10/1732124998230/Put/seqid=0 2024-11-20T17:49:59,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742306_1482 (size=12301) 2024-11-20T17:49:59,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/12feb2bc09cf4a529263ce4515f5e810 2024-11-20T17:49:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/256fea829a1249d79b20400d2b471e66 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/256fea829a1249d79b20400d2b471e66 2024-11-20T17:49:59,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/256fea829a1249d79b20400d2b471e66, entries=200, sequenceid=406, filesize=39.0 K 2024-11-20T17:49:59,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/54ed2aed2de34aa4bc4d1732e86e218a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/54ed2aed2de34aa4bc4d1732e86e218a 2024-11-20T17:49:59,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/54ed2aed2de34aa4bc4d1732e86e218a, entries=150, sequenceid=406, filesize=12.0 K 2024-11-20T17:49:59,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/12feb2bc09cf4a529263ce4515f5e810 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/12feb2bc09cf4a529263ce4515f5e810 2024-11-20T17:49:59,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/12feb2bc09cf4a529263ce4515f5e810, entries=150, sequenceid=406, filesize=12.0 K 2024-11-20T17:49:59,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d16d3ef4696b9dba5a878aefd3c902e4 in 1650ms, sequenceid=406, compaction requested=false 2024-11-20T17:49:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:50:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:00,430 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:00,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T17:50:00,430 INFO [Thread-1665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1665. 2024-11-20T17:50:00,430 INFO [Thread-1667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1667. 2024-11-20T17:50:00,430 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-11-20T17:50:00,430 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:00,430 DEBUG [Thread-1665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:00,430 DEBUG [Thread-1667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:00,430 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,430 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,430 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,430 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,430 DEBUG [Thread-1665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,430 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:00,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:00,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:00,433 INFO [Thread-1671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1671. 2024-11-20T17:50:00,433 DEBUG [Thread-1671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:00,433 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,433 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:00,439 INFO [Thread-1669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1669. 2024-11-20T17:50:00,439 DEBUG [Thread-1669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:00,439 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,439 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-20T17:50:00,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120801babec4fc740638444af38ff0a065f_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_0/A:col10/1732125000428/Put/seqid=0 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1186 2024-11-20T17:50:00,439 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3558 rows 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1187 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3561 rows 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1189 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3567 rows 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1184 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3552 rows 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1195 2024-11-20T17:50:00,440 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3585 rows 2024-11-20T17:50:00,440 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:50:00,440 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:50:00,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:00,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:00,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T17:50:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742307_1483 (size=12454) 2024-11-20T17:50:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:00,444 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125000444"}]},"ts":"1732125000444"} 2024-11-20T17:50:00,445 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T17:50:00,445 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T17:50:00,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:50:00,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, UNASSIGN}] 2024-11-20T17:50:00,447 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, UNASSIGN 2024-11-20T17:50:00,447 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:00,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, UNASSIGN because future has completed 2024-11-20T17:50:00,448 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:50:00,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:50:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:00,600 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:00,600 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:50:00,601 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing d16d3ef4696b9dba5a878aefd3c902e4, disabling compactions & flushes 2024-11-20T17:50:00,601 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:50:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:00,843 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:00,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120801babec4fc740638444af38ff0a065f_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120801babec4fc740638444af38ff0a065f_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:00,847 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/92774b00ca0e4f418b607dbf16c348ac, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:50:00,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/92774b00ca0e4f418b607dbf16c348ac is 175, key is test_row_0/A:col10/1732125000428/Put/seqid=0 2024-11-20T17:50:00,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742308_1484 (size=31255) 2024-11-20T17:50:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:01,251 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/92774b00ca0e4f418b607dbf16c348ac 2024-11-20T17:50:01,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/6526328d7b004eb0b524f4eea72a6de0 is 50, key is test_row_0/B:col10/1732125000428/Put/seqid=0 2024-11-20T17:50:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742309_1485 (size=12301) 2024-11-20T17:50:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:01,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/6526328d7b004eb0b524f4eea72a6de0 2024-11-20T17:50:01,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2d3f443c59a44e6faae332a9d79e6fa1 is 50, key is test_row_0/C:col10/1732125000428/Put/seqid=0 2024-11-20T17:50:01,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742310_1486 (size=12301) 2024-11-20T17:50:02,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2d3f443c59a44e6faae332a9d79e6fa1 2024-11-20T17:50:02,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/92774b00ca0e4f418b607dbf16c348ac as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/92774b00ca0e4f418b607dbf16c348ac 2024-11-20T17:50:02,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/92774b00ca0e4f418b607dbf16c348ac, entries=150, sequenceid=436, filesize=30.5 K 2024-11-20T17:50:02,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/6526328d7b004eb0b524f4eea72a6de0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/6526328d7b004eb0b524f4eea72a6de0 2024-11-20T17:50:02,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/6526328d7b004eb0b524f4eea72a6de0, entries=150, sequenceid=436, filesize=12.0 K 2024-11-20T17:50:02,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/2d3f443c59a44e6faae332a9d79e6fa1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2d3f443c59a44e6faae332a9d79e6fa1 2024-11-20T17:50:02,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2d3f443c59a44e6faae332a9d79e6fa1, entries=150, sequenceid=436, filesize=12.0 K 2024-11-20T17:50:02,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=6.71 KB/6870 for d16d3ef4696b9dba5a878aefd3c902e4 in 1656ms, sequenceid=436, compaction requested=true 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d16d3ef4696b9dba5a878aefd3c902e4: 2024-11-20T17:50:02,086 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:02,086 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:02,086 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. because compaction request was cancelled 2024-11-20T17:50:02,086 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. after waiting 0 ms 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:02,086 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:50:02,086 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:A 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:02,086 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. because compaction request was cancelled 2024-11-20T17:50:02,086 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:B 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d16d3ef4696b9dba5a878aefd3c902e4:C, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:02,086 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. because compaction request was cancelled 2024-11-20T17:50:02,086 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing d16d3ef4696b9dba5a878aefd3c902e4 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-20T17:50:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:02,086 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d16d3ef4696b9dba5a878aefd3c902e4:C 2024-11-20T17:50:02,090 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5eaba57aa544c17b0dfa78b1ea10f2b_d16d3ef4696b9dba5a878aefd3c902e4 is 50, key is test_row_1/A:col10/1732125000437/Put/seqid=0 2024-11-20T17:50:02,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742311_1487 (size=7374) 2024-11-20T17:50:02,494 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:02,497 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5eaba57aa544c17b0dfa78b1ea10f2b_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5eaba57aa544c17b0dfa78b1ea10f2b_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:02,497 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/0e7fe3d2d0c2473d99ce2db7d7db0199, store: [table=TestAcidGuarantees family=A region=d16d3ef4696b9dba5a878aefd3c902e4] 2024-11-20T17:50:02,498 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/0e7fe3d2d0c2473d99ce2db7d7db0199 is 175, key is test_row_1/A:col10/1732125000437/Put/seqid=0 2024-11-20T17:50:02,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742312_1488 (size=13865) 2024-11-20T17:50:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:02,902 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/0e7fe3d2d0c2473d99ce2db7d7db0199 2024-11-20T17:50:02,907 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/bcfb4a8d449944a6b5dcb34bd3ac453c is 50, key is test_row_1/B:col10/1732125000437/Put/seqid=0 2024-11-20T17:50:02,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742313_1489 (size=7415) 2024-11-20T17:50:03,311 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/bcfb4a8d449944a6b5dcb34bd3ac453c 2024-11-20T17:50:03,317 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/80e50b80aa1740c894817f2675f7dca0 is 50, key is test_row_1/C:col10/1732125000437/Put/seqid=0 2024-11-20T17:50:03,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742314_1490 (size=7415) 2024-11-20T17:50:03,722 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/80e50b80aa1740c894817f2675f7dca0 2024-11-20T17:50:03,727 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/A/0e7fe3d2d0c2473d99ce2db7d7db0199 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/0e7fe3d2d0c2473d99ce2db7d7db0199 2024-11-20T17:50:03,730 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/0e7fe3d2d0c2473d99ce2db7d7db0199, entries=50, sequenceid=440, filesize=13.5 K 2024-11-20T17:50:03,731 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/B/bcfb4a8d449944a6b5dcb34bd3ac453c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/bcfb4a8d449944a6b5dcb34bd3ac453c 2024-11-20T17:50:03,734 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/bcfb4a8d449944a6b5dcb34bd3ac453c, entries=50, sequenceid=440, filesize=7.2 K 2024-11-20T17:50:03,734 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/.tmp/C/80e50b80aa1740c894817f2675f7dca0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/80e50b80aa1740c894817f2675f7dca0 2024-11-20T17:50:03,737 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/80e50b80aa1740c894817f2675f7dca0, entries=50, sequenceid=440, filesize=7.2 K 2024-11-20T17:50:03,738 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d16d3ef4696b9dba5a878aefd3c902e4 in 1652ms, sequenceid=440, compaction requested=true 2024-11-20T17:50:03,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/514b93136fd34c2bb8c85b3b56bf59d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576] to archive 2024-11-20T17:50:03,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:50:03,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/514b93136fd34c2bb8c85b3b56bf59d9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/514b93136fd34c2bb8c85b3b56bf59d9 2024-11-20T17:50:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/7c2f4ccf799a4114b045657769c72645 2024-11-20T17:50:03,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/94b06347256b47efbebb5f29b2665109 2024-11-20T17:50:03,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/d8f7ca172d474a069a792a9b16202576 2024-11-20T17:50:03,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8349f3c2666849a5aac073404408fadc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/05f7a86c24544feca5d9b1d411ef67f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/07ddc183a539417cbf67dfb1b72f2e86, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/fe7296f82ce84204a274ffb4cf2b888b] to archive 2024-11-20T17:50:03,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:50:03,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8349f3c2666849a5aac073404408fadc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8349f3c2666849a5aac073404408fadc 2024-11-20T17:50:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/05f7a86c24544feca5d9b1d411ef67f9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/05f7a86c24544feca5d9b1d411ef67f9 2024-11-20T17:50:03,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/07ddc183a539417cbf67dfb1b72f2e86 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/07ddc183a539417cbf67dfb1b72f2e86 2024-11-20T17:50:03,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/fe7296f82ce84204a274ffb4cf2b888b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/fe7296f82ce84204a274ffb4cf2b888b 2024-11-20T17:50:03,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/49aa448621134b5e98b3693cdba52ce9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2632b14572744f4f9fc06abcf1e5ae06, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/61e711a58a214ac38b3587d11a9992f3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/ebc285115cd6433d85f8a5ca76593df9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d432673997a6426d8e07c1f0ba48864f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d702f8ebe540417cb3846b123efb1dc9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/1209a0726e0d4b4081f528d9f34f8e31] to archive 2024-11-20T17:50:03,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:50:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/49aa448621134b5e98b3693cdba52ce9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/49aa448621134b5e98b3693cdba52ce9 2024-11-20T17:50:03,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2632b14572744f4f9fc06abcf1e5ae06 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2632b14572744f4f9fc06abcf1e5ae06 2024-11-20T17:50:03,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/61e711a58a214ac38b3587d11a9992f3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/61e711a58a214ac38b3587d11a9992f3 2024-11-20T17:50:03,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/ebc285115cd6433d85f8a5ca76593df9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/ebc285115cd6433d85f8a5ca76593df9 2024-11-20T17:50:03,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d432673997a6426d8e07c1f0ba48864f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d432673997a6426d8e07c1f0ba48864f 2024-11-20T17:50:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d702f8ebe540417cb3846b123efb1dc9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/d702f8ebe540417cb3846b123efb1dc9 2024-11-20T17:50:03,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/1209a0726e0d4b4081f528d9f34f8e31 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/1209a0726e0d4b4081f528d9f34f8e31 2024-11-20T17:50:03,759 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/recovered.edits/443.seqid, newMaxSeqId=443, maxSeqId=4 2024-11-20T17:50:03,759 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. 2024-11-20T17:50:03,760 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for d16d3ef4696b9dba5a878aefd3c902e4: Waiting for close lock at 1732125000600Running coprocessor pre-close hooks at 1732125000600Disabling compacts and flushes for region at 1732125000600Disabling writes for close at 1732125002086 (+1486 ms)Obtaining lock to block concurrent updates at 1732125002086Preparing flush snapshotting stores in d16d3ef4696b9dba5a878aefd3c902e4 at 1732125002086Finished memstore snapshotting TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732125002086Flushing stores of TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4. at 1732125002087 (+1 ms)Flushing d16d3ef4696b9dba5a878aefd3c902e4/A: creating writer at 1732125002087Flushing mob file d16d3ef4696b9dba5a878aefd3c902e4/A: appending metadata at 1732125002090 (+3 ms)Flushing mob file d16d3ef4696b9dba5a878aefd3c902e4/A: closing flushed file at 1732125002090Flushing d16d3ef4696b9dba5a878aefd3c902e4/A: appending metadata at 1732125002498 (+408 ms)Flushing d16d3ef4696b9dba5a878aefd3c902e4/A: closing flushed file at 1732125002498Flushing d16d3ef4696b9dba5a878aefd3c902e4/B: creating writer at 1732125002905 (+407 ms)Flushing d16d3ef4696b9dba5a878aefd3c902e4/B: appending metadata at 1732125002907 (+2 ms)Flushing d16d3ef4696b9dba5a878aefd3c902e4/B: closing flushed file at 1732125002907Flushing d16d3ef4696b9dba5a878aefd3c902e4/C: creating writer at 1732125003314 (+407 ms)Flushing d16d3ef4696b9dba5a878aefd3c902e4/C: appending metadata at 1732125003316 (+2 ms)Flushing d16d3ef4696b9dba5a878aefd3c902e4/C: closing flushed file at 1732125003316Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c8ca7b2: reopening flushed file at 1732125003726 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d937306: reopening flushed file at 1732125003730 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d84b55e: reopening flushed file at 1732125003734 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for d16d3ef4696b9dba5a878aefd3c902e4 in 1652ms, sequenceid=440, compaction requested=true at 1732125003738 (+4 ms)Writing region close event to WAL at 1732125003756 (+18 ms)Running coprocessor post-close hooks at 1732125003759 (+3 ms)Closed at 1732125003759 2024-11-20T17:50:03,761 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:03,762 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=d16d3ef4696b9dba5a878aefd3c902e4, regionState=CLOSED 2024-11-20T17:50:03,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:50:03,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T17:50:03,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure d16d3ef4696b9dba5a878aefd3c902e4, server=613581fcdc40,44957,1732124872893 in 3.3160 sec 2024-11-20T17:50:03,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-20T17:50:03,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d16d3ef4696b9dba5a878aefd3c902e4, UNASSIGN in 3.3200 sec 2024-11-20T17:50:03,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T17:50:03,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3230 sec 2024-11-20T17:50:03,770 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125003770"}]},"ts":"1732125003770"} 2024-11-20T17:50:03,772 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T17:50:03,772 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T17:50:03,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3310 sec 2024-11-20T17:50:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T17:50:04,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:04,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T17:50:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,610 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T17:50:04,610 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,615 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/recovered.edits] 2024-11-20T17:50:04,617 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/0e7fe3d2d0c2473d99ce2db7d7db0199 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/0e7fe3d2d0c2473d99ce2db7d7db0199 2024-11-20T17:50:04,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/256fea829a1249d79b20400d2b471e66 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/256fea829a1249d79b20400d2b471e66 2024-11-20T17:50:04,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/92774b00ca0e4f418b607dbf16c348ac to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/92774b00ca0e4f418b607dbf16c348ac 2024-11-20T17:50:04,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/b5255371b6674c4d90b402cb49ddb03f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/A/b5255371b6674c4d90b402cb49ddb03f 2024-11-20T17:50:04,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/54ed2aed2de34aa4bc4d1732e86e218a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/54ed2aed2de34aa4bc4d1732e86e218a 2024-11-20T17:50:04,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/6526328d7b004eb0b524f4eea72a6de0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/6526328d7b004eb0b524f4eea72a6de0 2024-11-20T17:50:04,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8ed94142c2504d2b885b09ffc2b80286 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/8ed94142c2504d2b885b09ffc2b80286 2024-11-20T17:50:04,624 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/bcfb4a8d449944a6b5dcb34bd3ac453c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/B/bcfb4a8d449944a6b5dcb34bd3ac453c 2024-11-20T17:50:04,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/12feb2bc09cf4a529263ce4515f5e810 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/12feb2bc09cf4a529263ce4515f5e810 2024-11-20T17:50:04,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2d3f443c59a44e6faae332a9d79e6fa1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/2d3f443c59a44e6faae332a9d79e6fa1 2024-11-20T17:50:04,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/80e50b80aa1740c894817f2675f7dca0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/80e50b80aa1740c894817f2675f7dca0 2024-11-20T17:50:04,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c634d97028054047a14541c928e3fd18 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/C/c634d97028054047a14541c928e3fd18 2024-11-20T17:50:04,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/recovered.edits/443.seqid to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4/recovered.edits/443.seqid 2024-11-20T17:50:04,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,630 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T17:50:04,631 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T17:50:04,631 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T17:50:04,633 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201573d6959bd14fa699a311775847ed0b_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201573d6959bd14fa699a311775847ed0b_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201a79a42d82c54836a63246244abb2c79_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201a79a42d82c54836a63246244abb2c79_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201cfb53c9c6a74dd5833c12296bfd4cbc_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201cfb53c9c6a74dd5833c12296bfd4cbc_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202f5c51118dac48c7ae96592ca2ec630d_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202f5c51118dac48c7ae96592ca2ec630d_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203648e0a2370f4f5395a411ac83fe2d27_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203648e0a2370f4f5395a411ac83fe2d27_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203e78547a3a5249b0a1b35e1059ed09d6_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203e78547a3a5249b0a1b35e1059ed09d6_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205020a99f59a4438c85ade923c0516b1f_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205020a99f59a4438c85ade923c0516b1f_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112050f9f70411e34f59a3240b39297b4e43_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112050f9f70411e34f59a3240b39297b4e43_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112066a7f5aea1a04347ac90060ae7c86e4c_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112066a7f5aea1a04347ac90060ae7c86e4c_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206b7c2ac1a11c49c9b6910ad441dc31a1_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206b7c2ac1a11c49c9b6910ad441dc31a1_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207035cd47cba942bb87ada98f0775aa31_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207035cd47cba942bb87ada98f0775aa31_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120721c9ec5abb64851b46f307337a84e66_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120721c9ec5abb64851b46f307337a84e66_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207384c92a43824e95ad8d50487e375c69_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207384c92a43824e95ad8d50487e375c69_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112076cfef5ca6ed40499104fd0bafa67b9a_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112076cfef5ca6ed40499104fd0bafa67b9a_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120801babec4fc740638444af38ff0a065f_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120801babec4fc740638444af38ff0a065f_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208ba3961649dd4263a5608f087baecd36_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208ba3961649dd4263a5608f087baecd36_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5eaba57aa544c17b0dfa78b1ea10f2b_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5eaba57aa544c17b0dfa78b1ea10f2b_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b7847b192e8142f889d5930c0cab2223_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b7847b192e8142f889d5930c0cab2223_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d83c50e6b2334919981e47d1aa897594_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d83c50e6b2334919981e47d1aa897594_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d9efaf56bd384a47803de28ae0a9efcf_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d9efaf56bd384a47803de28ae0a9efcf_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4c461a44c5447aba8f9c7d51fc67f60_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4c461a44c5447aba8f9c7d51fc67f60_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f2e03eeedaa5477f8930fa7e65a24701_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f2e03eeedaa5477f8930fa7e65a24701_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f6654f96692346a6962e76ae742f54c8_d16d3ef4696b9dba5a878aefd3c902e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f6654f96692346a6962e76ae742f54c8_d16d3ef4696b9dba5a878aefd3c902e4 2024-11-20T17:50:04,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T17:50:04,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,654 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T17:50:04,656 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T17:50:04,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T17:50:04,657 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732125004657"}]},"ts":"9223372036854775807"} 2024-11-20T17:50:04,659 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T17:50:04,659 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d16d3ef4696b9dba5a878aefd3c902e4, NAME => 'TestAcidGuarantees,,1732124975133.d16d3ef4696b9dba5a878aefd3c902e4.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T17:50:04,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T17:50:04,659 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732125004659"}]},"ts":"9223372036854775807"} 2024-11-20T17:50:04,660 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T17:50:04,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-20T17:50:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T17:50:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T17:50:04,718 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:04,727 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 236) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-141067869_22 at /127.0.0.1:33910 [Waiting for operation #1023] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-945899218_22 at /127.0.0.1:43642 [Waiting for operation #609] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-945899218_22 at /127.0.0.1:40552 [Waiting for operation #702] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-141067869_22 at /127.0.0.1:43652 [Waiting for operation #604] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 433) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8185 (was 7917) - AvailableMemoryMB LEAK? - 2024-11-20T17:50:04,735 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=8185 2024-11-20T17:50:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:50:04,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:50:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:04,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:50:04,739 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:04,739 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-20T17:50:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-20T17:50:04,740 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:50:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742315_1491 (size=959) 2024-11-20T17:50:04,748 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:50:04,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742316_1492 (size=53) 2024-11-20T17:50:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-20T17:50:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-20T17:50:05,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:50:05,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0db4ba01acc0539d81a46ec1c40b3d92, disabling compactions & flushes 2024-11-20T17:50:05,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. after waiting 0 ms 2024-11-20T17:50:05,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0db4ba01acc0539d81a46ec1c40b3d92: Waiting for close lock at 1732125005153Disabling compacts and flushes for region at 1732125005153Disabling writes for close at 1732125005153Writing region close event to WAL at 1732125005153Closed at 1732125005153 2024-11-20T17:50:05,154 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:50:05,155 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732125005155"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125005155"}]},"ts":"1732125005155"} 2024-11-20T17:50:05,156 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:50:05,157 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:50:05,157 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125005157"}]},"ts":"1732125005157"} 2024-11-20T17:50:05,158 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T17:50:05,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, ASSIGN}] 2024-11-20T17:50:05,160 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, ASSIGN 2024-11-20T17:50:05,161 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, ASSIGN; state=OFFLINE, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=false 2024-11-20T17:50:05,311 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=0db4ba01acc0539d81a46ec1c40b3d92, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, ASSIGN because future has completed 2024-11-20T17:50:05,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:50:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-20T17:50:05,467 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,468 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:50:05,468 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,468 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:50:05,468 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,468 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,469 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,470 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0db4ba01acc0539d81a46ec1c40b3d92 columnFamilyName A 2024-11-20T17:50:05,470 DEBUG [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:05,471 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] regionserver.HStore(327): Store=0db4ba01acc0539d81a46ec1c40b3d92/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:05,471 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,472 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0db4ba01acc0539d81a46ec1c40b3d92 columnFamilyName B 2024-11-20T17:50:05,472 DEBUG [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:05,472 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] regionserver.HStore(327): Store=0db4ba01acc0539d81a46ec1c40b3d92/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:05,472 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,473 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0db4ba01acc0539d81a46ec1c40b3d92 columnFamilyName C 2024-11-20T17:50:05,473 DEBUG [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:05,473 INFO [StoreOpener-0db4ba01acc0539d81a46ec1c40b3d92-1 {}] regionserver.HStore(327): Store=0db4ba01acc0539d81a46ec1c40b3d92/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:05,473 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,474 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,474 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,475 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,475 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,475 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:50:05,476 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,477 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:50:05,478 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 0db4ba01acc0539d81a46ec1c40b3d92; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68112203, jitterRate=0.014950916171073914}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:50:05,478 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,478 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 0db4ba01acc0539d81a46ec1c40b3d92: Running coprocessor pre-open hook at 1732125005468Writing region info on filesystem at 1732125005468Initializing all the Stores at 1732125005469 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125005469Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125005469Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125005469Cleaning up temporary data from old regions at 1732125005475 (+6 ms)Running coprocessor post-open hooks at 1732125005478 (+3 ms)Region opened successfully at 1732125005478 2024-11-20T17:50:05,479 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., pid=124, masterSystemTime=1732125005465 2024-11-20T17:50:05,480 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,480 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:05,481 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=0db4ba01acc0539d81a46ec1c40b3d92, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:50:05,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T17:50:05,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 in 170 msec 2024-11-20T17:50:05,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-20T17:50:05,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, ASSIGN in 325 msec 2024-11-20T17:50:05,486 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:50:05,486 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125005486"}]},"ts":"1732125005486"} 2024-11-20T17:50:05,487 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T17:50:05,488 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:50:05,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 751 msec 2024-11-20T17:50:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-20T17:50:05,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:05,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16654396, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d4939a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209534fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,884 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:50:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:50:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@de824ec 2024-11-20T17:50:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:50:05,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:50:05,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57af2b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32057d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,888 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1ddccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ae5a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d964a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1574de36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fe770e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,893 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4fdfd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,894 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdfecc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cc712e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19caab6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295eafd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6196a59c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,898 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21cd061e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca0cf46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b7256f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6609511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,902 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee3946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@188c1103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,903 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51377d1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,905 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651e5ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f46cad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,907 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179972dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,908 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e769a60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d22d375, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e9e7a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be413b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b6e143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:05,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:05,912 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d5fe61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:05,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:05,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12fdba7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:05,914 DEBUG [Thread-2201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,914 DEBUG [Thread-2203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,914 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,914 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,915 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,915 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:05,915 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,916 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,916 DEBUG [Thread-2199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-20T17:50:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,917 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,917 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:05,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:05,921 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,922 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:05,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2] 2024-11-20T17:50:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:05,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:05,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:50:05,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/73d90c44905149e19023fa9a377ae007 is 50, key is test_row_0/A:col10/1732125005929/Put/seqid=0 2024-11-20T17:50:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125065964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:05,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742317_1493 (size=12001) 2024-11-20T17:50:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125065966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125065966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125065967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125065967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:06,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125066077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125066087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125066087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125066087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125066088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:06,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125066287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125066297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125066297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125066297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125066297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/73d90c44905149e19023fa9a377ae007 2024-11-20T17:50:06,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/ff073f401b384a3a960f1eb02894a431 is 50, key is test_row_0/B:col10/1732125005929/Put/seqid=0 2024-11-20T17:50:06,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742318_1494 (size=12001) 2024-11-20T17:50:06,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:06,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125066597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125066607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125066607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125066607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125066607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:06,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/ff073f401b384a3a960f1eb02894a431 2024-11-20T17:50:06,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/45f23935856f4ca0b69f43eb9bea47d4 is 50, key is test_row_0/C:col10/1732125005929/Put/seqid=0 2024-11-20T17:50:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742319_1495 (size=12001) 2024-11-20T17:50:06,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:06,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125067107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:07,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125067117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125067117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125067117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:07,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125067117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:07,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:07,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:07,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:07,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:07,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:07,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:07,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/45f23935856f4ca0b69f43eb9bea47d4 2024-11-20T17:50:07,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/73d90c44905149e19023fa9a377ae007 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/73d90c44905149e19023fa9a377ae007 2024-11-20T17:50:07,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/73d90c44905149e19023fa9a377ae007, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:50:07,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/ff073f401b384a3a960f1eb02894a431 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/ff073f401b384a3a960f1eb02894a431 2024-11-20T17:50:07,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/ff073f401b384a3a960f1eb02894a431, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:50:07,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/45f23935856f4ca0b69f43eb9bea47d4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/45f23935856f4ca0b69f43eb9bea47d4 2024-11-20T17:50:07,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/45f23935856f4ca0b69f43eb9bea47d4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T17:50:07,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1313ms, sequenceid=13, compaction requested=false 2024-11-20T17:50:07,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:07,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T17:50:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:07,298 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:50:07,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/5b00ff2e33904a1c8bc1319cf3bb45c7 is 50, key is test_row_0/A:col10/1732125005965/Put/seqid=0 2024-11-20T17:50:07,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742320_1496 (size=12001) 2024-11-20T17:50:07,706 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/5b00ff2e33904a1c8bc1319cf3bb45c7 2024-11-20T17:50:07,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/e5b2b87f83004411936ab69d5475e56f is 50, key is test_row_0/B:col10/1732125005965/Put/seqid=0 2024-11-20T17:50:07,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742321_1497 (size=12001) 2024-11-20T17:50:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:08,116 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/e5b2b87f83004411936ab69d5475e56f 2024-11-20T17:50:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/8d307f7e6f424ab688145efa4444bdce is 50, key is test_row_0/C:col10/1732125005965/Put/seqid=0 2024-11-20T17:50:08,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742322_1498 (size=12001) 2024-11-20T17:50:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:08,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:08,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125068142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125068143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125068144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125068145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125068148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125068257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125068257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125068257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125068258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125068258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125068467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125068467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125068467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125068468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125068468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/8d307f7e6f424ab688145efa4444bdce 2024-11-20T17:50:08,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/5b00ff2e33904a1c8bc1319cf3bb45c7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5b00ff2e33904a1c8bc1319cf3bb45c7 2024-11-20T17:50:08,533 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5b00ff2e33904a1c8bc1319cf3bb45c7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T17:50:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/e5b2b87f83004411936ab69d5475e56f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e5b2b87f83004411936ab69d5475e56f 2024-11-20T17:50:08,537 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e5b2b87f83004411936ab69d5475e56f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T17:50:08,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/8d307f7e6f424ab688145efa4444bdce as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/8d307f7e6f424ab688145efa4444bdce 2024-11-20T17:50:08,540 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/8d307f7e6f424ab688145efa4444bdce, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T17:50:08,541 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1243ms, sequenceid=37, compaction requested=false 2024-11-20T17:50:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-20T17:50:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-20T17:50:08,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T17:50:08,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6200 sec 2024-11-20T17:50:08,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6290 sec 2024-11-20T17:50:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:08,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:08,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:50:08,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/118453d303eb4ce792d3edca20aa76de is 50, key is test_row_0/A:col10/1732125008141/Put/seqid=0 2024-11-20T17:50:08,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742323_1499 (size=16681) 2024-11-20T17:50:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125068810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125068811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125068818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125068819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125068822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125068927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125068927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125068927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125068937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125068937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/118453d303eb4ce792d3edca20aa76de 2024-11-20T17:50:09,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2f8736fb51654aedad18426efd8a1772 is 50, key is test_row_0/B:col10/1732125008141/Put/seqid=0 2024-11-20T17:50:09,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742324_1500 (size=12001) 2024-11-20T17:50:09,280 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:50:09,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125069456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125069457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125069457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125069457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125069467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:09,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:09,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2f8736fb51654aedad18426efd8a1772 2024-11-20T17:50:09,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/64512161c4f943a495354612c35b6501 is 50, key is test_row_0/C:col10/1732125008141/Put/seqid=0 2024-11-20T17:50:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742325_1501 (size=12001) 2024-11-20T17:50:09,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/64512161c4f943a495354612c35b6501 2024-11-20T17:50:09,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/118453d303eb4ce792d3edca20aa76de as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/118453d303eb4ce792d3edca20aa76de 2024-11-20T17:50:09,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/118453d303eb4ce792d3edca20aa76de, entries=250, sequenceid=51, filesize=16.3 K 2024-11-20T17:50:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2f8736fb51654aedad18426efd8a1772 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2f8736fb51654aedad18426efd8a1772 2024-11-20T17:50:09,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2f8736fb51654aedad18426efd8a1772, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T17:50:09,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/64512161c4f943a495354612c35b6501 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/64512161c4f943a495354612c35b6501 2024-11-20T17:50:09,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/64512161c4f943a495354612c35b6501, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T17:50:09,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0db4ba01acc0539d81a46ec1c40b3d92 in 858ms, sequenceid=51, compaction requested=true 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:09,638 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:09,638 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:09,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:09,639 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:09,639 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:09,639 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/73d90c44905149e19023fa9a377ae007, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5b00ff2e33904a1c8bc1319cf3bb45c7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/118453d303eb4ce792d3edca20aa76de] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=39.7 K 2024-11-20T17:50:09,639 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/ff073f401b384a3a960f1eb02894a431, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e5b2b87f83004411936ab69d5475e56f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2f8736fb51654aedad18426efd8a1772] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=35.2 K 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73d90c44905149e19023fa9a377ae007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732125005929 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ff073f401b384a3a960f1eb02894a431, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732125005929 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b00ff2e33904a1c8bc1319cf3bb45c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732125005964 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b2b87f83004411936ab69d5475e56f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732125005964 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 118453d303eb4ce792d3edca20aa76de, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732125008141 2024-11-20T17:50:09,639 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f8736fb51654aedad18426efd8a1772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732125008141 2024-11-20T17:50:09,646 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:09,646 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0447ab6f092441a9bbbd5aa83f681e86 is 50, key is test_row_0/A:col10/1732125008141/Put/seqid=0 2024-11-20T17:50:09,656 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:09,657 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/a566111f565646b0a842aa399adb87e4 is 50, key is test_row_0/B:col10/1732125008141/Put/seqid=0 2024-11-20T17:50:09,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742326_1502 (size=12104) 2024-11-20T17:50:09,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742327_1503 (size=12104) 2024-11-20T17:50:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:09,972 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:09,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:50:09,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0fd373c088f4485597358392d230223a is 50, key is test_row_0/A:col10/1732125008810/Put/seqid=0 2024-11-20T17:50:09,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742328_1504 (size=14341) 2024-11-20T17:50:09,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125069989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125069994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125069995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125069999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125070000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,060 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0447ab6f092441a9bbbd5aa83f681e86 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0447ab6f092441a9bbbd5aa83f681e86 2024-11-20T17:50:10,064 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/a566111f565646b0a842aa399adb87e4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a566111f565646b0a842aa399adb87e4 2024-11-20T17:50:10,064 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into 0447ab6f092441a9bbbd5aa83f681e86(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:10,064 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:10,064 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125009638; duration=0sec 2024-11-20T17:50:10,064 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:10,064 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:10,064 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:10,065 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:10,065 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:10,065 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,065 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/45f23935856f4ca0b69f43eb9bea47d4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/8d307f7e6f424ab688145efa4444bdce, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/64512161c4f943a495354612c35b6501] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=35.2 K 2024-11-20T17:50:10,065 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f23935856f4ca0b69f43eb9bea47d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732125005929 2024-11-20T17:50:10,066 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d307f7e6f424ab688145efa4444bdce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732125005964 2024-11-20T17:50:10,066 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64512161c4f943a495354612c35b6501, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732125008141 2024-11-20T17:50:10,067 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into a566111f565646b0a842aa399adb87e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:10,067 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:10,067 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125009638; duration=0sec 2024-11-20T17:50:10,067 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:10,067 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:10,070 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#419 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:10,071 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9f2473ea2d5f4318975959812678829a is 50, key is test_row_0/C:col10/1732125008141/Put/seqid=0 2024-11-20T17:50:10,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742329_1505 (size=12104) 2024-11-20T17:50:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T17:50:10,077 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:10,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-20T17:50:10,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T17:50:10,079 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:10,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125070107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125070107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125070107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125070117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125070117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T17:50:10,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T17:50:10,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:10,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125070317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125070317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125070317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125070327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125070327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T17:50:10,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0fd373c088f4485597358392d230223a 2024-11-20T17:50:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/e10e275f65314b5a8ddfe3736565b6f9 is 50, key is test_row_0/B:col10/1732125008810/Put/seqid=0 2024-11-20T17:50:10,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742330_1506 (size=12001) 2024-11-20T17:50:10,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/e10e275f65314b5a8ddfe3736565b6f9 2024-11-20T17:50:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T17:50:10,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/21c59d59dd5e4c0193c63b5cb8d47cb7 is 50, key is test_row_0/C:col10/1732125008810/Put/seqid=0 2024-11-20T17:50:10,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742331_1507 (size=12001) 2024-11-20T17:50:10,481 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9f2473ea2d5f4318975959812678829a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9f2473ea2d5f4318975959812678829a 2024-11-20T17:50:10,485 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into 9f2473ea2d5f4318975959812678829a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:10,485 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:10,485 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125009638; duration=0sec 2024-11-20T17:50:10,485 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:10,485 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:10,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T17:50:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125070627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125070628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125070628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125070637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125070637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:10,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T17:50:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T17:50:10,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/21c59d59dd5e4c0193c63b5cb8d47cb7 2024-11-20T17:50:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0fd373c088f4485597358392d230223a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0fd373c088f4485597358392d230223a 2024-11-20T17:50:10,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0fd373c088f4485597358392d230223a, entries=200, sequenceid=74, filesize=14.0 K 2024-11-20T17:50:10,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/e10e275f65314b5a8ddfe3736565b6f9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e10e275f65314b5a8ddfe3736565b6f9 2024-11-20T17:50:10,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e10e275f65314b5a8ddfe3736565b6f9, entries=150, sequenceid=74, filesize=11.7 K 2024-11-20T17:50:10,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/21c59d59dd5e4c0193c63b5cb8d47cb7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/21c59d59dd5e4c0193c63b5cb8d47cb7 2024-11-20T17:50:10,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/21c59d59dd5e4c0193c63b5cb8d47cb7, entries=150, sequenceid=74, filesize=11.7 K 2024-11-20T17:50:10,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0db4ba01acc0539d81a46ec1c40b3d92 in 865ms, sequenceid=74, compaction requested=false 2024-11-20T17:50:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:10,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T17:50:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:10,843 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:50:10,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/f30c740e0b094e4b9d44892649c8981e is 50, key is test_row_0/A:col10/1732125009998/Put/seqid=0 2024-11-20T17:50:10,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742332_1508 (size=12001) 2024-11-20T17:50:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:11,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125071179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125071180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125071188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125071188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125071189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T17:50:11,254 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/f30c740e0b094e4b9d44892649c8981e 2024-11-20T17:50:11,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/d661b3da3e914382b159faea2a340487 is 50, key is test_row_0/B:col10/1732125009998/Put/seqid=0 2024-11-20T17:50:11,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742333_1509 (size=12001) 2024-11-20T17:50:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125071297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125071298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125071298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125071308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125071309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125071507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125071517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125071517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125071518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125071518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,670 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/d661b3da3e914382b159faea2a340487 2024-11-20T17:50:11,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/0c89bbb215ab4f1db8f6dae32b742c6c is 50, key is test_row_0/C:col10/1732125009998/Put/seqid=0 2024-11-20T17:50:11,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742334_1510 (size=12001) 2024-11-20T17:50:11,730 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/0c89bbb215ab4f1db8f6dae32b742c6c 2024-11-20T17:50:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/f30c740e0b094e4b9d44892649c8981e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f30c740e0b094e4b9d44892649c8981e 2024-11-20T17:50:11,760 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f30c740e0b094e4b9d44892649c8981e, entries=150, sequenceid=90, filesize=11.7 K 2024-11-20T17:50:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/d661b3da3e914382b159faea2a340487 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d661b3da3e914382b159faea2a340487 2024-11-20T17:50:11,769 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d661b3da3e914382b159faea2a340487, entries=150, sequenceid=90, filesize=11.7 K 2024-11-20T17:50:11,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/0c89bbb215ab4f1db8f6dae32b742c6c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c89bbb215ab4f1db8f6dae32b742c6c 2024-11-20T17:50:11,812 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c89bbb215ab4f1db8f6dae32b742c6c, entries=150, sequenceid=90, filesize=11.7 K 2024-11-20T17:50:11,817 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0db4ba01acc0539d81a46ec1c40b3d92 in 973ms, sequenceid=90, compaction requested=true 2024-11-20T17:50:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:11,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-20T17:50:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-20T17:50:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:11,823 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:11,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:50:11,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T17:50:11,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-20T17:50:11,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7470 sec 2024-11-20T17:50:11,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0d978066a33c4dacafe2531701d38e3e is 50, key is test_row_0/A:col10/1732125011188/Put/seqid=0 2024-11-20T17:50:11,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742335_1511 (size=14341) 2024-11-20T17:50:11,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0d978066a33c4dacafe2531701d38e3e 2024-11-20T17:50:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125071844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125071847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125071852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/69e791c5b5984f65a76905d13c1341d0 is 50, key is test_row_0/B:col10/1732125011188/Put/seqid=0 2024-11-20T17:50:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125071854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125071856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742336_1512 (size=12001) 2024-11-20T17:50:11,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125071968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125071970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125071970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125071978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125071979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125072177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125072185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125072187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125072187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125072187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T17:50:12,227 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:12,227 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-20T17:50:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T17:50:12,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:12,230 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:12,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:12,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/69e791c5b5984f65a76905d13c1341d0 2024-11-20T17:50:12,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/a77b4a2edd894f56b4d29526866711ae is 50, key is test_row_0/C:col10/1732125011188/Put/seqid=0 2024-11-20T17:50:12,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742337_1513 (size=12001) 2024-11-20T17:50:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T17:50:12,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T17:50:12,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125072497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125072499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125072507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125072507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125072507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:12,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T17:50:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T17:50:12,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T17:50:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:12,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/a77b4a2edd894f56b4d29526866711ae 2024-11-20T17:50:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0d978066a33c4dacafe2531701d38e3e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0d978066a33c4dacafe2531701d38e3e 2024-11-20T17:50:12,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0d978066a33c4dacafe2531701d38e3e, entries=200, sequenceid=114, filesize=14.0 K 2024-11-20T17:50:12,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/69e791c5b5984f65a76905d13c1341d0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/69e791c5b5984f65a76905d13c1341d0 2024-11-20T17:50:12,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/69e791c5b5984f65a76905d13c1341d0, entries=150, sequenceid=114, filesize=11.7 K 2024-11-20T17:50:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/a77b4a2edd894f56b4d29526866711ae as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/a77b4a2edd894f56b4d29526866711ae 2024-11-20T17:50:12,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/a77b4a2edd894f56b4d29526866711ae, entries=150, sequenceid=114, filesize=11.7 K 2024-11-20T17:50:12,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0db4ba01acc0539d81a46ec1c40b3d92 in 948ms, sequenceid=114, compaction requested=true 2024-11-20T17:50:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:12,772 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:50:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:12,772 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:50:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:12,774 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:50:12,774 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:50:12,774 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:12,774 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:12,774 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,774 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,774 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a566111f565646b0a842aa399adb87e4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e10e275f65314b5a8ddfe3736565b6f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d661b3da3e914382b159faea2a340487, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/69e791c5b5984f65a76905d13c1341d0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=47.0 K 2024-11-20T17:50:12,774 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0447ab6f092441a9bbbd5aa83f681e86, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0fd373c088f4485597358392d230223a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f30c740e0b094e4b9d44892649c8981e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0d978066a33c4dacafe2531701d38e3e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=51.5 K 2024-11-20T17:50:12,775 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a566111f565646b0a842aa399adb87e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732125008141 2024-11-20T17:50:12,775 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0447ab6f092441a9bbbd5aa83f681e86, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732125008141 2024-11-20T17:50:12,775 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e10e275f65314b5a8ddfe3736565b6f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732125008810 2024-11-20T17:50:12,775 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd373c088f4485597358392d230223a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732125008810 2024-11-20T17:50:12,775 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d661b3da3e914382b159faea2a340487, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732125009993 2024-11-20T17:50:12,775 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f30c740e0b094e4b9d44892649c8981e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732125009993 2024-11-20T17:50:12,776 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 69e791c5b5984f65a76905d13c1341d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732125011186 2024-11-20T17:50:12,776 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d978066a33c4dacafe2531701d38e3e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732125011178 2024-11-20T17:50:12,793 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#428 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:12,793 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8d1e632a8b794694a6c52c4e71841458 is 50, key is test_row_0/B:col10/1732125011188/Put/seqid=0 2024-11-20T17:50:12,806 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:12,806 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/f532a0bd2f674a5babf3a4a894c1c9cd is 50, key is test_row_0/A:col10/1732125011188/Put/seqid=0 2024-11-20T17:50:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742338_1514 (size=12241) 2024-11-20T17:50:12,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T17:50:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:50:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T17:50:12,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/1c6eea99c4be455d98edc8840784011e is 50, key is test_row_0/A:col10/1732125011853/Put/seqid=0 2024-11-20T17:50:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742339_1515 (size=12241) 2024-11-20T17:50:12,869 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/f532a0bd2f674a5babf3a4a894c1c9cd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f532a0bd2f674a5babf3a4a894c1c9cd 2024-11-20T17:50:12,885 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into f532a0bd2f674a5babf3a4a894c1c9cd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:12,885 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:12,885 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=12, startTime=1732125012772; duration=0sec 2024-11-20T17:50:12,885 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:12,885 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:12,885 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:50:12,886 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:50:12,886 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:12,886 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:12,889 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9f2473ea2d5f4318975959812678829a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/21c59d59dd5e4c0193c63b5cb8d47cb7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c89bbb215ab4f1db8f6dae32b742c6c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/a77b4a2edd894f56b4d29526866711ae] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=47.0 K 2024-11-20T17:50:12,890 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2473ea2d5f4318975959812678829a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732125008141 2024-11-20T17:50:12,890 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c59d59dd5e4c0193c63b5cb8d47cb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732125008810 2024-11-20T17:50:12,890 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c89bbb215ab4f1db8f6dae32b742c6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732125009993 2024-11-20T17:50:12,890 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77b4a2edd894f56b4d29526866711ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732125011186 2024-11-20T17:50:12,904 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#431 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:12,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742340_1516 (size=12001) 2024-11-20T17:50:12,905 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/2e628440ffe444f78c98781f7fd35b87 is 50, key is test_row_0/C:col10/1732125011188/Put/seqid=0 2024-11-20T17:50:12,906 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/1c6eea99c4be455d98edc8840784011e 2024-11-20T17:50:12,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/9036b8c43ec9402093944e687d3dac77 is 50, key is test_row_0/B:col10/1732125011853/Put/seqid=0 2024-11-20T17:50:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742341_1517 (size=12241) 2024-11-20T17:50:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742342_1518 (size=12001) 2024-11-20T17:50:12,968 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/9036b8c43ec9402093944e687d3dac77 2024-11-20T17:50:12,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/0c825b613ce441c8883819ef4aa19a89 is 50, key is test_row_0/C:col10/1732125011853/Put/seqid=0 2024-11-20T17:50:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742343_1519 (size=12001) 2024-11-20T17:50:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:13,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:13,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125073111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125073113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125073115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125073124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125073125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125073237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125073238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125073238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,250 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8d1e632a8b794694a6c52c4e71841458 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8d1e632a8b794694a6c52c4e71841458 2024-11-20T17:50:13,255 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into 8d1e632a8b794694a6c52c4e71841458(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:13,255 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,255 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=12, startTime=1732125012772; duration=0sec 2024-11-20T17:50:13,255 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:13,255 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:13,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125073248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,363 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/2e628440ffe444f78c98781f7fd35b87 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/2e628440ffe444f78c98781f7fd35b87 2024-11-20T17:50:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T17:50:13,369 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into 2e628440ffe444f78c98781f7fd35b87(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:13,369 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,369 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=12, startTime=1732125012772; duration=0sec 2024-11-20T17:50:13,369 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:13,369 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:13,428 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/0c825b613ce441c8883819ef4aa19a89 2024-11-20T17:50:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/1c6eea99c4be455d98edc8840784011e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/1c6eea99c4be455d98edc8840784011e 2024-11-20T17:50:13,440 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/1c6eea99c4be455d98edc8840784011e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T17:50:13,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/9036b8c43ec9402093944e687d3dac77 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9036b8c43ec9402093944e687d3dac77 2024-11-20T17:50:13,445 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9036b8c43ec9402093944e687d3dac77, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T17:50:13,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/0c825b613ce441c8883819ef4aa19a89 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c825b613ce441c8883819ef4aa19a89 2024-11-20T17:50:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c825b613ce441c8883819ef4aa19a89, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T17:50:13,452 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0db4ba01acc0539d81a46ec1c40b3d92 in 601ms, sequenceid=126, compaction requested=false 2024-11-20T17:50:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-20T17:50:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-20T17:50:13,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T17:50:13,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2240 sec 2024-11-20T17:50:13,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.2300 sec 2024-11-20T17:50:13,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:13,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T17:50:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:13,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/51a44a1c1a0f41c782f97507147121cb is 50, key is test_row_0/A:col10/1732125013116/Put/seqid=0 2024-11-20T17:50:13,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125073478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125073485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125073485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125073486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125073487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742344_1520 (size=14541) 2024-11-20T17:50:13,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/51a44a1c1a0f41c782f97507147121cb 2024-11-20T17:50:13,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/c55f884d98484de2a0ff30881d91fc62 is 50, key is test_row_0/B:col10/1732125013116/Put/seqid=0 2024-11-20T17:50:13,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742345_1521 (size=12151) 2024-11-20T17:50:13,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/c55f884d98484de2a0ff30881d91fc62 2024-11-20T17:50:13,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/71dce67c4bf0492aa488b67790e0f91e is 50, key is test_row_0/C:col10/1732125013116/Put/seqid=0 2024-11-20T17:50:13,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742346_1522 (size=12151) 2024-11-20T17:50:13,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/71dce67c4bf0492aa488b67790e0f91e 2024-11-20T17:50:13,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/51a44a1c1a0f41c782f97507147121cb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/51a44a1c1a0f41c782f97507147121cb 2024-11-20T17:50:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125073607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125073607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125073607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125073608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/51a44a1c1a0f41c782f97507147121cb, entries=200, sequenceid=155, filesize=14.2 K 2024-11-20T17:50:13,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/c55f884d98484de2a0ff30881d91fc62 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/c55f884d98484de2a0ff30881d91fc62 2024-11-20T17:50:13,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/c55f884d98484de2a0ff30881d91fc62, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T17:50:13,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/71dce67c4bf0492aa488b67790e0f91e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/71dce67c4bf0492aa488b67790e0f91e 2024-11-20T17:50:13,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/71dce67c4bf0492aa488b67790e0f91e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T17:50:13,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0db4ba01acc0539d81a46ec1c40b3d92 in 162ms, sequenceid=155, compaction requested=true 2024-11-20T17:50:13,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,631 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:13,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:13,632 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:13,632 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:13,632 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:13,632 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f532a0bd2f674a5babf3a4a894c1c9cd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/1c6eea99c4be455d98edc8840784011e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/51a44a1c1a0f41c782f97507147121cb] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=37.9 K 2024-11-20T17:50:13,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:13,633 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:13,634 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f532a0bd2f674a5babf3a4a894c1c9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732125011186 2024-11-20T17:50:13,634 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c6eea99c4be455d98edc8840784011e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732125011829 2024-11-20T17:50:13,634 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a44a1c1a0f41c782f97507147121cb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732125013111 2024-11-20T17:50:13,635 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:13,635 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:13,635 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:13,635 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8d1e632a8b794694a6c52c4e71841458, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9036b8c43ec9402093944e687d3dac77, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/c55f884d98484de2a0ff30881d91fc62] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=35.5 K 2024-11-20T17:50:13,635 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1e632a8b794694a6c52c4e71841458, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732125011186 2024-11-20T17:50:13,636 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9036b8c43ec9402093944e687d3dac77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732125011829 2024-11-20T17:50:13,637 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c55f884d98484de2a0ff30881d91fc62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732125013111 2024-11-20T17:50:13,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:13,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:13,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:13,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:13,658 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:13,659 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/cd9a3a1be4af4294b84bc5a1465a9e33 is 50, key is test_row_0/A:col10/1732125013116/Put/seqid=0 2024-11-20T17:50:13,663 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:13,663 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/a529d153da874b958756525747443333 is 50, key is test_row_0/B:col10/1732125013116/Put/seqid=0 2024-11-20T17:50:13,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742347_1523 (size=12493) 2024-11-20T17:50:13,700 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/cd9a3a1be4af4294b84bc5a1465a9e33 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cd9a3a1be4af4294b84bc5a1465a9e33 2024-11-20T17:50:13,705 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into cd9a3a1be4af4294b84bc5a1465a9e33(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:13,705 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,705 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125013631; duration=0sec 2024-11-20T17:50:13,705 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:13,705 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:13,705 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:13,707 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:13,707 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:13,707 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:13,707 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/2e628440ffe444f78c98781f7fd35b87, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c825b613ce441c8883819ef4aa19a89, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/71dce67c4bf0492aa488b67790e0f91e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=35.5 K 2024-11-20T17:50:13,708 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e628440ffe444f78c98781f7fd35b87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732125011186 2024-11-20T17:50:13,708 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c825b613ce441c8883819ef4aa19a89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732125011829 2024-11-20T17:50:13,708 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71dce67c4bf0492aa488b67790e0f91e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732125013111 2024-11-20T17:50:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742348_1524 (size=12493) 2024-11-20T17:50:13,716 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/a529d153da874b958756525747443333 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a529d153da874b958756525747443333 2024-11-20T17:50:13,724 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:13,724 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/59a4a9f8391d4804bc2ca387a53a1564 is 50, key is test_row_0/C:col10/1732125013116/Put/seqid=0 2024-11-20T17:50:13,726 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into a529d153da874b958756525747443333(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:13,726 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,726 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125013633; duration=0sec 2024-11-20T17:50:13,726 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:13,726 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:13,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742349_1525 (size=12493) 2024-11-20T17:50:13,738 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/59a4a9f8391d4804bc2ca387a53a1564 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/59a4a9f8391d4804bc2ca387a53a1564 2024-11-20T17:50:13,765 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into 59a4a9f8391d4804bc2ca387a53a1564(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:13,765 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:13,765 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125013644; duration=0sec 2024-11-20T17:50:13,765 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:13,765 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:13,801 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:13,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:50:13,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/713a47f786c14350a335c9655ff38684 is 50, key is test_row_0/A:col10/1732125013480/Put/seqid=0 2024-11-20T17:50:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742350_1526 (size=14541) 2024-11-20T17:50:13,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125073866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125073868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125073870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125073873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125073880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125073987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125073987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125073987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125073988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125073997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125074198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125074198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125074198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125074198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125074209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/713a47f786c14350a335c9655ff38684 2024-11-20T17:50:14,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/b43fd927da214421a05fb32188e7fb95 is 50, key is test_row_0/B:col10/1732125013480/Put/seqid=0 2024-11-20T17:50:14,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742351_1527 (size=12151) 2024-11-20T17:50:14,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/b43fd927da214421a05fb32188e7fb95 2024-11-20T17:50:14,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/63630a72efea414b803079645a9100dc is 50, key is test_row_0/C:col10/1732125013480/Put/seqid=0 2024-11-20T17:50:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742352_1528 (size=12151) 2024-11-20T17:50:14,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/63630a72efea414b803079645a9100dc 2024-11-20T17:50:14,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/713a47f786c14350a335c9655ff38684 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/713a47f786c14350a335c9655ff38684 2024-11-20T17:50:14,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/713a47f786c14350a335c9655ff38684, entries=200, sequenceid=169, filesize=14.2 K 2024-11-20T17:50:14,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/b43fd927da214421a05fb32188e7fb95 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b43fd927da214421a05fb32188e7fb95 2024-11-20T17:50:14,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b43fd927da214421a05fb32188e7fb95, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T17:50:14,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/63630a72efea414b803079645a9100dc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/63630a72efea414b803079645a9100dc 2024-11-20T17:50:14,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/63630a72efea414b803079645a9100dc, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T17:50:14,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0db4ba01acc0539d81a46ec1c40b3d92 in 501ms, sequenceid=169, compaction requested=false 2024-11-20T17:50:14,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T17:50:14,377 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:14,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-20T17:50:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T17:50:14,380 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:14,383 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:14,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T17:50:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:14,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:14,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T17:50:14,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/20bf01131cab4fda917388392090b880 is 50, key is test_row_0/A:col10/1732125013849/Put/seqid=0 2024-11-20T17:50:14,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742353_1529 (size=14541) 2024-11-20T17:50:14,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125074520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125074527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125074528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T17:50:14,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:14,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125074529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125074529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125074639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125074639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125074639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125074647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125074647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T17:50:14,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T17:50:14,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T17:50:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125074858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125074858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125074858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125074858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125074860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:14,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/20bf01131cab4fda917388392090b880 2024-11-20T17:50:14,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2749c4ed726245e59456acbf7ae9a008 is 50, key is test_row_0/B:col10/1732125013849/Put/seqid=0 2024-11-20T17:50:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742354_1530 (size=12151) 2024-11-20T17:50:14,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2749c4ed726245e59456acbf7ae9a008 2024-11-20T17:50:14,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/f887e0372b5743d586342f6fd1e5c58f is 50, key is test_row_0/C:col10/1732125013849/Put/seqid=0 2024-11-20T17:50:14,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742355_1531 (size=12151) 2024-11-20T17:50:14,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/f887e0372b5743d586342f6fd1e5c58f 2024-11-20T17:50:14,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/20bf01131cab4fda917388392090b880 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/20bf01131cab4fda917388392090b880 2024-11-20T17:50:14,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/20bf01131cab4fda917388392090b880, entries=200, sequenceid=195, filesize=14.2 K 2024-11-20T17:50:14,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2749c4ed726245e59456acbf7ae9a008 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2749c4ed726245e59456acbf7ae9a008 2024-11-20T17:50:14,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2749c4ed726245e59456acbf7ae9a008, entries=150, sequenceid=195, filesize=11.9 K 2024-11-20T17:50:14,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/f887e0372b5743d586342f6fd1e5c58f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f887e0372b5743d586342f6fd1e5c58f 2024-11-20T17:50:14,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T17:50:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:14,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T17:50:15,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f887e0372b5743d586342f6fd1e5c58f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-20T17:50:15,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0db4ba01acc0539d81a46ec1c40b3d92 in 496ms, sequenceid=195, compaction requested=true 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:15,010 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:15,010 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:15,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:15,011 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:15,011 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:15,011 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:15,011 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:15,011 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:15,011 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:15,011 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cd9a3a1be4af4294b84bc5a1465a9e33, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/713a47f786c14350a335c9655ff38684, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/20bf01131cab4fda917388392090b880] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=40.6 K 2024-11-20T17:50:15,011 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a529d153da874b958756525747443333, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b43fd927da214421a05fb32188e7fb95, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2749c4ed726245e59456acbf7ae9a008] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=35.9 K 2024-11-20T17:50:15,011 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd9a3a1be4af4294b84bc5a1465a9e33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732125013111 2024-11-20T17:50:15,011 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting a529d153da874b958756525747443333, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732125013111 2024-11-20T17:50:15,012 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713a47f786c14350a335c9655ff38684, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732125013480 2024-11-20T17:50:15,012 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b43fd927da214421a05fb32188e7fb95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732125013480 2024-11-20T17:50:15,012 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20bf01131cab4fda917388392090b880, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732125013849 2024-11-20T17:50:15,012 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2749c4ed726245e59456acbf7ae9a008, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732125013849 2024-11-20T17:50:15,020 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:15,021 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/6ffa161873ce4f38841dfa314bfc1b73 is 50, key is test_row_0/B:col10/1732125013849/Put/seqid=0 2024-11-20T17:50:15,023 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:15,024 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/66a853671f924ab785ae768f8784ad8e is 50, key is test_row_0/A:col10/1732125013849/Put/seqid=0 2024-11-20T17:50:15,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742356_1532 (size=12595) 2024-11-20T17:50:15,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742357_1533 (size=12595) 2024-11-20T17:50:15,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T17:50:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:15,154 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T17:50:15,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/fade9ea3d56541e09bd4bf47c086556f is 50, key is test_row_0/A:col10/1732125014526/Put/seqid=0 2024-11-20T17:50:15,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742358_1534 (size=12151) 2024-11-20T17:50:15,175 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/fade9ea3d56541e09bd4bf47c086556f 2024-11-20T17:50:15,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/6a6b4adff21f435b8fb239bef7b6bda7 is 50, key is test_row_0/B:col10/1732125014526/Put/seqid=0 2024-11-20T17:50:15,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742359_1535 (size=12151) 2024-11-20T17:50:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125075229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125075230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125075231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125075239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125075243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125075347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125075347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125075347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125075347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125075357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,437 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/6ffa161873ce4f38841dfa314bfc1b73 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6ffa161873ce4f38841dfa314bfc1b73 2024-11-20T17:50:15,442 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into 6ffa161873ce4f38841dfa314bfc1b73(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:15,442 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:15,442 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125015010; duration=0sec 2024-11-20T17:50:15,443 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:15,443 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:15,443 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:15,443 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:15,443 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:15,443 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:15,443 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/59a4a9f8391d4804bc2ca387a53a1564, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/63630a72efea414b803079645a9100dc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f887e0372b5743d586342f6fd1e5c58f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=35.9 K 2024-11-20T17:50:15,444 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a4a9f8391d4804bc2ca387a53a1564, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732125013111 2024-11-20T17:50:15,444 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 63630a72efea414b803079645a9100dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732125013480 2024-11-20T17:50:15,444 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f887e0372b5743d586342f6fd1e5c58f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732125013849 2024-11-20T17:50:15,445 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/66a853671f924ab785ae768f8784ad8e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/66a853671f924ab785ae768f8784ad8e 2024-11-20T17:50:15,453 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into 66a853671f924ab785ae768f8784ad8e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:15,453 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:15,453 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125015010; duration=0sec 2024-11-20T17:50:15,453 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:15,453 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:15,455 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:15,456 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9791bdee6d9c4a14b3e034547c98f77d is 50, key is test_row_0/C:col10/1732125013849/Put/seqid=0 2024-11-20T17:50:15,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742360_1536 (size=12595) 2024-11-20T17:50:15,464 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9791bdee6d9c4a14b3e034547c98f77d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9791bdee6d9c4a14b3e034547c98f77d 2024-11-20T17:50:15,468 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into 9791bdee6d9c4a14b3e034547c98f77d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:15,468 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:15,468 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125015010; duration=0sec 2024-11-20T17:50:15,468 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:15,468 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T17:50:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125075567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125075567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125075567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125075568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125075578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,602 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/6a6b4adff21f435b8fb239bef7b6bda7 2024-11-20T17:50:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/f7670b1dd7b64bb0ba662d8a729ab2e4 is 50, key is test_row_0/C:col10/1732125014526/Put/seqid=0 2024-11-20T17:50:15,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742361_1537 (size=12151) 2024-11-20T17:50:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125075878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125075878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125075887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125075887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125075887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:15,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:15,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,019 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/f7670b1dd7b64bb0ba662d8a729ab2e4 2024-11-20T17:50:16,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/fade9ea3d56541e09bd4bf47c086556f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/fade9ea3d56541e09bd4bf47c086556f 2024-11-20T17:50:16,026 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/fade9ea3d56541e09bd4bf47c086556f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T17:50:16,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/6a6b4adff21f435b8fb239bef7b6bda7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6a6b4adff21f435b8fb239bef7b6bda7 2024-11-20T17:50:16,031 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6a6b4adff21f435b8fb239bef7b6bda7, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T17:50:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/f7670b1dd7b64bb0ba662d8a729ab2e4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f7670b1dd7b64bb0ba662d8a729ab2e4 2024-11-20T17:50:16,035 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f7670b1dd7b64bb0ba662d8a729ab2e4, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T17:50:16,036 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0db4ba01acc0539d81a46ec1c40b3d92 in 882ms, sequenceid=205, compaction requested=false 2024-11-20T17:50:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-20T17:50:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-20T17:50:16,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T17:50:16,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6540 sec 2024-11-20T17:50:16,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6610 sec 2024-11-20T17:50:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:16,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T17:50:16,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7bdbe31898064e1aa44110a04c7d9926 is 50, key is test_row_0/A:col10/1732125015229/Put/seqid=0 2024-11-20T17:50:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742362_1538 (size=16931) 2024-11-20T17:50:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125076411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125076411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125076413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125076417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125076418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T17:50:16,527 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:16,528 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-20T17:50:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T17:50:16,529 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:16,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:16,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125076527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125076528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125076528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125076537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125076537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T17:50:16,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125076737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125076747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125076747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125076747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125076747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:16,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7bdbe31898064e1aa44110a04c7d9926 2024-11-20T17:50:16,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/1596ee8897024d6e9f3d7447f9717894 is 50, key is test_row_0/B:col10/1732125015229/Put/seqid=0 2024-11-20T17:50:16,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742363_1539 (size=12151) 2024-11-20T17:50:16,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:16,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T17:50:16,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:16,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125077047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125077057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125077057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125077057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125077057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:17,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:17,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T17:50:17,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/1596ee8897024d6e9f3d7447f9717894 2024-11-20T17:50:17,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/65707b2ee2f54bc296c383203eb3ebd4 is 50, key is test_row_0/C:col10/1732125015229/Put/seqid=0 2024-11-20T17:50:17,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742364_1540 (size=12151) 2024-11-20T17:50:17,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:17,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125077567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125077577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125077577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125077577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125077577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:17,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:17,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/65707b2ee2f54bc296c383203eb3ebd4 2024-11-20T17:50:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7bdbe31898064e1aa44110a04c7d9926 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7bdbe31898064e1aa44110a04c7d9926 2024-11-20T17:50:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T17:50:17,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7bdbe31898064e1aa44110a04c7d9926, entries=250, sequenceid=236, filesize=16.5 K 2024-11-20T17:50:17,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/1596ee8897024d6e9f3d7447f9717894 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1596ee8897024d6e9f3d7447f9717894 2024-11-20T17:50:17,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1596ee8897024d6e9f3d7447f9717894, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T17:50:17,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/65707b2ee2f54bc296c383203eb3ebd4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/65707b2ee2f54bc296c383203eb3ebd4 2024-11-20T17:50:17,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/65707b2ee2f54bc296c383203eb3ebd4, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T17:50:17,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1286ms, sequenceid=236, compaction requested=true 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:17,682 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:50:17,682 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:17,686 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:17,686 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:17,686 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,686 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6ffa161873ce4f38841dfa314bfc1b73, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6a6b4adff21f435b8fb239bef7b6bda7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1596ee8897024d6e9f3d7447f9717894] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.0 K 2024-11-20T17:50:17,686 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:17,686 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:17,686 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,686 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/66a853671f924ab785ae768f8784ad8e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/fade9ea3d56541e09bd4bf47c086556f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7bdbe31898064e1aa44110a04c7d9926] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=40.7 K 2024-11-20T17:50:17,687 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ffa161873ce4f38841dfa314bfc1b73, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732125013849 2024-11-20T17:50:17,687 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a853671f924ab785ae768f8784ad8e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732125013849 2024-11-20T17:50:17,687 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a6b4adff21f435b8fb239bef7b6bda7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732125014526 2024-11-20T17:50:17,687 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fade9ea3d56541e09bd4bf47c086556f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732125014526 2024-11-20T17:50:17,687 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1596ee8897024d6e9f3d7447f9717894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732125015229 2024-11-20T17:50:17,688 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bdbe31898064e1aa44110a04c7d9926, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732125015229 2024-11-20T17:50:17,702 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:17,703 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/9e1175f6f01948f7b1658f4b47809c29 is 50, key is test_row_0/B:col10/1732125015229/Put/seqid=0 2024-11-20T17:50:17,711 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:17,711 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/5fd66b8037774b459d0db4484e2662db is 50, key is test_row_0/A:col10/1732125015229/Put/seqid=0 2024-11-20T17:50:17,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742365_1541 (size=12697) 2024-11-20T17:50:17,733 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/9e1175f6f01948f7b1658f4b47809c29 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9e1175f6f01948f7b1658f4b47809c29 2024-11-20T17:50:17,740 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into 9e1175f6f01948f7b1658f4b47809c29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:17,740 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:17,740 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125017682; duration=0sec 2024-11-20T17:50:17,740 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:17,740 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:17,741 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:17,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742366_1542 (size=12697) 2024-11-20T17:50:17,744 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:17,744 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:17,745 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,745 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9791bdee6d9c4a14b3e034547c98f77d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f7670b1dd7b64bb0ba662d8a729ab2e4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/65707b2ee2f54bc296c383203eb3ebd4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.0 K 2024-11-20T17:50:17,745 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9791bdee6d9c4a14b3e034547c98f77d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732125013849 2024-11-20T17:50:17,747 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f7670b1dd7b64bb0ba662d8a729ab2e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732125014526 2024-11-20T17:50:17,748 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 65707b2ee2f54bc296c383203eb3ebd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732125015229 2024-11-20T17:50:17,749 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/5fd66b8037774b459d0db4484e2662db as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5fd66b8037774b459d0db4484e2662db 2024-11-20T17:50:17,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T17:50:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,756 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T17:50:17,763 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into 5fd66b8037774b459d0db4484e2662db(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:17,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:17,763 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125017682; duration=0sec 2024-11-20T17:50:17,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:17,763 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:17,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/9a736130ae4a4c478e692faf2263131e is 50, key is test_row_1/A:col10/1732125016411/Put/seqid=0 2024-11-20T17:50:17,765 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:17,765 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/6b53ef1a1c1b45588f02338afd7fc60d is 50, key is test_row_0/C:col10/1732125015229/Put/seqid=0 2024-11-20T17:50:17,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742367_1543 (size=9757) 2024-11-20T17:50:17,804 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/9a736130ae4a4c478e692faf2263131e 2024-11-20T17:50:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8104755200b7480581ebe6f103e5d6c3 is 50, key is test_row_1/B:col10/1732125016411/Put/seqid=0 2024-11-20T17:50:17,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742368_1544 (size=12697) 2024-11-20T17:50:17,827 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/6b53ef1a1c1b45588f02338afd7fc60d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6b53ef1a1c1b45588f02338afd7fc60d 2024-11-20T17:50:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742369_1545 (size=9757) 2024-11-20T17:50:17,831 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8104755200b7480581ebe6f103e5d6c3 2024-11-20T17:50:17,838 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into 6b53ef1a1c1b45588f02338afd7fc60d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:17,838 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:17,838 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125017682; duration=0sec 2024-11-20T17:50:17,838 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:17,838 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:17,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/734572c3f6e2416cbdfecadd04cb8490 is 50, key is test_row_1/C:col10/1732125016411/Put/seqid=0 2024-11-20T17:50:17,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742370_1546 (size=9757) 2024-11-20T17:50:17,856 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/734572c3f6e2416cbdfecadd04cb8490 2024-11-20T17:50:17,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/9a736130ae4a4c478e692faf2263131e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/9a736130ae4a4c478e692faf2263131e 2024-11-20T17:50:17,868 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/9a736130ae4a4c478e692faf2263131e, entries=100, sequenceid=245, filesize=9.5 K 2024-11-20T17:50:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8104755200b7480581ebe6f103e5d6c3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8104755200b7480581ebe6f103e5d6c3 2024-11-20T17:50:17,874 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8104755200b7480581ebe6f103e5d6c3, entries=100, sequenceid=245, filesize=9.5 K 2024-11-20T17:50:17,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/734572c3f6e2416cbdfecadd04cb8490 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/734572c3f6e2416cbdfecadd04cb8490 2024-11-20T17:50:17,882 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/734572c3f6e2416cbdfecadd04cb8490, entries=100, sequenceid=245, filesize=9.5 K 2024-11-20T17:50:17,883 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0db4ba01acc0539d81a46ec1c40b3d92 in 127ms, sequenceid=245, compaction requested=false 2024-11-20T17:50:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:17,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-20T17:50:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-20T17:50:17,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T17:50:17,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3540 sec 2024-11-20T17:50:17,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3590 sec 2024-11-20T17:50:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:18,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:18,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:50:18,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/17bcdb2f68f642e59570cfbfec37da31 is 50, key is test_row_0/A:col10/1732125018589/Put/seqid=0 2024-11-20T17:50:18,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742371_1547 (size=17031) 2024-11-20T17:50:18,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/17bcdb2f68f642e59570cfbfec37da31 2024-11-20T17:50:18,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2b527fe7b4c3460ea1b900f192a0b3d9 is 50, key is test_row_0/B:col10/1732125018589/Put/seqid=0 2024-11-20T17:50:18,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742372_1548 (size=12251) 2024-11-20T17:50:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125078638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125078639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125078646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125078650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125078651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T17:50:18,687 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:18,688 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-20T17:50:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T17:50:18,689 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:18,691 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:18,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125078757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125078757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125078758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125078767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125078768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T17:50:18,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T17:50:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:18,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:18,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125078977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125078977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125078977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125078978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125078978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:18,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T17:50:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:18,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T17:50:19,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2b527fe7b4c3460ea1b900f192a0b3d9 2024-11-20T17:50:19,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/745d9e83f4104b2e9a4edc22851e8b85 is 50, key is test_row_0/C:col10/1732125018589/Put/seqid=0 2024-11-20T17:50:19,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742373_1549 (size=12251) 2024-11-20T17:50:19,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T17:50:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125079287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125079287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125079287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125079287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125079297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T17:50:19,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:19,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T17:50:19,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/745d9e83f4104b2e9a4edc22851e8b85 2024-11-20T17:50:19,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/17bcdb2f68f642e59570cfbfec37da31 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/17bcdb2f68f642e59570cfbfec37da31 2024-11-20T17:50:19,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/17bcdb2f68f642e59570cfbfec37da31, entries=250, sequenceid=258, filesize=16.6 K 2024-11-20T17:50:19,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2b527fe7b4c3460ea1b900f192a0b3d9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2b527fe7b4c3460ea1b900f192a0b3d9 2024-11-20T17:50:19,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T17:50:19,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:19,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2b527fe7b4c3460ea1b900f192a0b3d9, entries=150, sequenceid=258, filesize=12.0 K 2024-11-20T17:50:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:19,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/745d9e83f4104b2e9a4edc22851e8b85 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/745d9e83f4104b2e9a4edc22851e8b85 2024-11-20T17:50:19,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/745d9e83f4104b2e9a4edc22851e8b85, entries=150, sequenceid=258, filesize=12.0 K 2024-11-20T17:50:19,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0db4ba01acc0539d81a46ec1c40b3d92 in 869ms, sequenceid=258, compaction requested=true 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:19,469 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:19,469 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:19,470 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:19,470 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39485 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:19,470 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:19,470 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:19,470 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,470 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5fd66b8037774b459d0db4484e2662db, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/9a736130ae4a4c478e692faf2263131e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/17bcdb2f68f642e59570cfbfec37da31] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=38.6 K 2024-11-20T17:50:19,470 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,470 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9e1175f6f01948f7b1658f4b47809c29, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8104755200b7480581ebe6f103e5d6c3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2b527fe7b4c3460ea1b900f192a0b3d9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=33.9 K 2024-11-20T17:50:19,470 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd66b8037774b459d0db4484e2662db, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732125015229 2024-11-20T17:50:19,472 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1175f6f01948f7b1658f4b47809c29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732125015229 2024-11-20T17:50:19,472 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a736130ae4a4c478e692faf2263131e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732125016411 2024-11-20T17:50:19,472 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8104755200b7480581ebe6f103e5d6c3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732125016411 2024-11-20T17:50:19,473 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17bcdb2f68f642e59570cfbfec37da31, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732125018584 2024-11-20T17:50:19,473 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b527fe7b4c3460ea1b900f192a0b3d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732125018589 2024-11-20T17:50:19,480 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:19,481 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/cf74a909f1e6446abc4aef20bcfa2da0 is 50, key is test_row_0/A:col10/1732125018589/Put/seqid=0 2024-11-20T17:50:19,489 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:19,490 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/b8b9e970c7ac457e9bb9da5e9e30468f is 50, key is test_row_0/B:col10/1732125018589/Put/seqid=0 2024-11-20T17:50:19,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742374_1550 (size=12899) 2024-11-20T17:50:19,496 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/cf74a909f1e6446abc4aef20bcfa2da0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cf74a909f1e6446abc4aef20bcfa2da0 2024-11-20T17:50:19,500 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into cf74a909f1e6446abc4aef20bcfa2da0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:19,500 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:19,500 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125019469; duration=0sec 2024-11-20T17:50:19,500 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:19,500 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:19,501 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:19,501 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:19,501 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:19,502 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,502 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6b53ef1a1c1b45588f02338afd7fc60d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/734572c3f6e2416cbdfecadd04cb8490, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/745d9e83f4104b2e9a4edc22851e8b85] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=33.9 K 2024-11-20T17:50:19,502 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b53ef1a1c1b45588f02338afd7fc60d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732125015229 2024-11-20T17:50:19,502 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734572c3f6e2416cbdfecadd04cb8490, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732125016411 2024-11-20T17:50:19,502 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 745d9e83f4104b2e9a4edc22851e8b85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732125018589 2024-11-20T17:50:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742375_1551 (size=12899) 2024-11-20T17:50:19,514 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:19,514 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/e891ccfe55234f3087e27acba13c7793 is 50, key is test_row_0/C:col10/1732125018589/Put/seqid=0 2024-11-20T17:50:19,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742376_1552 (size=12899) 2024-11-20T17:50:19,538 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/e891ccfe55234f3087e27acba13c7793 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e891ccfe55234f3087e27acba13c7793 2024-11-20T17:50:19,550 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into e891ccfe55234f3087e27acba13c7793(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:19,550 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:19,550 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125019469; duration=0sec 2024-11-20T17:50:19,551 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:19,551 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:19,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T17:50:19,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:19,616 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:50:19,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/65faa5993d5d44a3b930941776afdb2e is 50, key is test_row_0/A:col10/1732125018650/Put/seqid=0 2024-11-20T17:50:19,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742377_1553 (size=12301) 2024-11-20T17:50:19,642 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/65faa5993d5d44a3b930941776afdb2e 2024-11-20T17:50:19,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/7d0903a974b44465ab3b62ef4ba04800 is 50, key is test_row_0/B:col10/1732125018650/Put/seqid=0 2024-11-20T17:50:19,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742378_1554 (size=12301) 2024-11-20T17:50:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:19,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:19,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125079816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125079817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T17:50:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125079821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125079821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125079822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,914 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/b8b9e970c7ac457e9bb9da5e9e30468f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b8b9e970c7ac457e9bb9da5e9e30468f 2024-11-20T17:50:19,918 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into b8b9e970c7ac457e9bb9da5e9e30468f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:19,918 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:19,918 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125019469; duration=0sec 2024-11-20T17:50:19,918 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:19,918 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125079928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125079937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125079937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125079937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,064 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/7d0903a974b44465ab3b62ef4ba04800 2024-11-20T17:50:20,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/b1023ee4d73046229c1f8771d8972ea0 is 50, key is test_row_0/C:col10/1732125018650/Put/seqid=0 2024-11-20T17:50:20,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742379_1555 (size=12301) 2024-11-20T17:50:20,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125080137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125080147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125080147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125080147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125080457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125080457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125080457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125080467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,496 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/b1023ee4d73046229c1f8771d8972ea0 2024-11-20T17:50:20,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/65faa5993d5d44a3b930941776afdb2e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/65faa5993d5d44a3b930941776afdb2e 2024-11-20T17:50:20,503 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/65faa5993d5d44a3b930941776afdb2e, entries=150, sequenceid=285, filesize=12.0 K 2024-11-20T17:50:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/7d0903a974b44465ab3b62ef4ba04800 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/7d0903a974b44465ab3b62ef4ba04800 2024-11-20T17:50:20,509 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/7d0903a974b44465ab3b62ef4ba04800, entries=150, sequenceid=285, filesize=12.0 K 2024-11-20T17:50:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/b1023ee4d73046229c1f8771d8972ea0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b1023ee4d73046229c1f8771d8972ea0 2024-11-20T17:50:20,513 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b1023ee4d73046229c1f8771d8972ea0, entries=150, sequenceid=285, filesize=12.0 K 2024-11-20T17:50:20,514 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0db4ba01acc0539d81a46ec1c40b3d92 in 899ms, sequenceid=285, compaction requested=false 2024-11-20T17:50:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-20T17:50:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-20T17:50:20,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T17:50:20,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8240 sec 2024-11-20T17:50:20,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8290 sec 2024-11-20T17:50:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:20,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:50:20,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0b63c8cebe8a4fec8619f9603604a6b1 is 50, key is test_row_0/A:col10/1732125019820/Put/seqid=0 2024-11-20T17:50:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T17:50:20,847 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:20,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-20T17:50:20,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T17:50:20,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:20,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742380_1556 (size=14741) 2024-11-20T17:50:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T17:50:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125080977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125080984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125080984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125080985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125080986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T17:50:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125081097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125081107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125081107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125081108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T17:50:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T17:50:21,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:50:21,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0b63c8cebe8a4fec8619f9603604a6b1 2024-11-20T17:50:21,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/bcba3cc755c3404b9ce558ee7c5c8562 is 50, key is test_row_0/B:col10/1732125019820/Put/seqid=0 2024-11-20T17:50:21,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742381_1557 (size=12301) 2024-11-20T17:50:21,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T17:50:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125081307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125081319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125081319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125081319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T17:50:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T17:50:21,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T17:50:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125081628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125081637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125081637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125081637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:21,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/bcba3cc755c3404b9ce558ee7c5c8562 2024-11-20T17:50:21,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/cac25bf0b3a348b890302e3a7bf4fc6a is 50, key is test_row_0/C:col10/1732125019820/Put/seqid=0 2024-11-20T17:50:21,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742382_1558 (size=12301) 2024-11-20T17:50:21,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/cac25bf0b3a348b890302e3a7bf4fc6a 2024-11-20T17:50:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/0b63c8cebe8a4fec8619f9603604a6b1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0b63c8cebe8a4fec8619f9603604a6b1 2024-11-20T17:50:21,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0b63c8cebe8a4fec8619f9603604a6b1, entries=200, sequenceid=298, filesize=14.4 K 2024-11-20T17:50:21,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/bcba3cc755c3404b9ce558ee7c5c8562 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/bcba3cc755c3404b9ce558ee7c5c8562 2024-11-20T17:50:21,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/bcba3cc755c3404b9ce558ee7c5c8562, entries=150, sequenceid=298, filesize=12.0 K 2024-11-20T17:50:21,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/cac25bf0b3a348b890302e3a7bf4fc6a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/cac25bf0b3a348b890302e3a7bf4fc6a 2024-11-20T17:50:21,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/cac25bf0b3a348b890302e3a7bf4fc6a, entries=150, sequenceid=298, filesize=12.0 K 2024-11-20T17:50:21,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0db4ba01acc0539d81a46ec1c40b3d92 in 862ms, sequenceid=298, compaction requested=true 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:21,702 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:21,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:50:21,704 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:21,704 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:21,704 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:21,705 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,705 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cf74a909f1e6446abc4aef20bcfa2da0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/65faa5993d5d44a3b930941776afdb2e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0b63c8cebe8a4fec8619f9603604a6b1] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=39.0 K 2024-11-20T17:50:21,708 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting cf74a909f1e6446abc4aef20bcfa2da0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732125018589 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 65faa5993d5d44a3b930941776afdb2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732125018644 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:21,710 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b63c8cebe8a4fec8619f9603604a6b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732125019813 2024-11-20T17:50:21,710 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b8b9e970c7ac457e9bb9da5e9e30468f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/7d0903a974b44465ab3b62ef4ba04800, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/bcba3cc755c3404b9ce558ee7c5c8562] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.6 K 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b9e970c7ac457e9bb9da5e9e30468f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732125018589 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0903a974b44465ab3b62ef4ba04800, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732125018644 2024-11-20T17:50:21,710 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcba3cc755c3404b9ce558ee7c5c8562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732125019820 2024-11-20T17:50:21,715 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:21,715 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/144307fba5d0410c882c964b33d78bf4 is 50, key is test_row_0/A:col10/1732125019820/Put/seqid=0 2024-11-20T17:50:21,722 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:21,722 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/1ef895a07f0348899fe7116984ef230f is 50, key is test_row_0/B:col10/1732125019820/Put/seqid=0 2024-11-20T17:50:21,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742383_1559 (size=13051) 2024-11-20T17:50:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742384_1560 (size=13051) 2024-11-20T17:50:21,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T17:50:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:21,769 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:50:21,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/bf0b9d8b262f426aa455b216560b5af7 is 50, key is test_row_0/A:col10/1732125020985/Put/seqid=0 2024-11-20T17:50:21,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742385_1561 (size=12301) 2024-11-20T17:50:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T17:50:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125082073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,138 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/144307fba5d0410c882c964b33d78bf4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/144307fba5d0410c882c964b33d78bf4 2024-11-20T17:50:22,142 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/1ef895a07f0348899fe7116984ef230f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1ef895a07f0348899fe7116984ef230f 2024-11-20T17:50:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,144 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into 144307fba5d0410c882c964b33d78bf4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:22,144 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125082137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,144 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125021702; duration=0sec 2024-11-20T17:50:22,144 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:22,144 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,144 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,145 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:22,145 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:22,145 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:22,145 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e891ccfe55234f3087e27acba13c7793, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b1023ee4d73046229c1f8771d8972ea0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/cac25bf0b3a348b890302e3a7bf4fc6a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.6 K 2024-11-20T17:50:22,146 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into 1ef895a07f0348899fe7116984ef230f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:22,146 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:22,146 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125021702; duration=0sec 2024-11-20T17:50:22,146 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:22,146 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:22,146 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e891ccfe55234f3087e27acba13c7793, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732125018589 2024-11-20T17:50:22,146 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b1023ee4d73046229c1f8771d8972ea0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732125018644 2024-11-20T17:50:22,146 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting cac25bf0b3a348b890302e3a7bf4fc6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732125019820 2024-11-20T17:50:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125082148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125082148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,158 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:22,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125082148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,158 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/6a930c6c4a35425992a8dc9a96eb90c7 is 50, key is test_row_0/C:col10/1732125019820/Put/seqid=0 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742386_1562 (size=13051) 2024-11-20T17:50:22,174 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/6a930c6c4a35425992a8dc9a96eb90c7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6a930c6c4a35425992a8dc9a96eb90c7 2024-11-20T17:50:22,180 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into 6a930c6c4a35425992a8dc9a96eb90c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:22,180 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:22,180 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125021702; duration=0sec 2024-11-20T17:50:22,180 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:22,180 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:22,189 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/bf0b9d8b262f426aa455b216560b5af7 2024-11-20T17:50:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125082187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/78c83fa283f949be857df049f0c0194d is 50, key is test_row_0/B:col10/1732125020985/Put/seqid=0 2024-11-20T17:50:22,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742387_1563 (size=12301) 2024-11-20T17:50:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125082407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,604 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/78c83fa283f949be857df049f0c0194d 2024-11-20T17:50:22,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/684bc584119c41feb0f53a72895d7339 is 50, key is test_row_0/C:col10/1732125020985/Put/seqid=0 2024-11-20T17:50:22,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742388_1564 (size=12301) 2024-11-20T17:50:22,623 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/684bc584119c41feb0f53a72895d7339 2024-11-20T17:50:22,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/bf0b9d8b262f426aa455b216560b5af7 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/bf0b9d8b262f426aa455b216560b5af7 2024-11-20T17:50:22,633 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/bf0b9d8b262f426aa455b216560b5af7, entries=150, sequenceid=322, filesize=12.0 K 2024-11-20T17:50:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/78c83fa283f949be857df049f0c0194d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78c83fa283f949be857df049f0c0194d 2024-11-20T17:50:22,643 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78c83fa283f949be857df049f0c0194d, entries=150, sequenceid=322, filesize=12.0 K 2024-11-20T17:50:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/684bc584119c41feb0f53a72895d7339 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/684bc584119c41feb0f53a72895d7339 2024-11-20T17:50:22,651 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/684bc584119c41feb0f53a72895d7339, entries=150, sequenceid=322, filesize=12.0 K 2024-11-20T17:50:22,653 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0db4ba01acc0539d81a46ec1c40b3d92 in 883ms, sequenceid=322, compaction requested=false 2024-11-20T17:50:22,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:22,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:22,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-20T17:50:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-20T17:50:22,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T17:50:22,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-20T17:50:22,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.8100 sec 2024-11-20T17:50:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:22,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:50:22,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/c52543f2e4f5499d88fcd682ec9b2d0d is 50, key is test_row_0/A:col10/1732125022072/Put/seqid=0 2024-11-20T17:50:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742389_1565 (size=14741) 2024-11-20T17:50:22,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/c52543f2e4f5499d88fcd682ec9b2d0d 2024-11-20T17:50:22,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8ea930d11d2e42dfa45e643f266172ca is 50, key is test_row_0/B:col10/1732125022072/Put/seqid=0 2024-11-20T17:50:22,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742390_1566 (size=12301) 2024-11-20T17:50:22,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125082860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125082977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:22,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T17:50:22,998 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:22,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-20T17:50:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T17:50:23,000 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:23,001 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:23,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T17:50:23,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T17:50:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8ea930d11d2e42dfa45e643f266172ca 2024-11-20T17:50:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125083157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:23,159 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:23,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/26779ff971904888ba58b6b2df407064 is 50, key is test_row_0/C:col10/1732125022072/Put/seqid=0 2024-11-20T17:50:23,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742391_1567 (size=12301) 2024-11-20T17:50:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125083167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:23,174 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125083169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:23,178 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:23,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:23,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125083178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:23,184 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:23,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125083187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:23,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T17:50:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T17:50:23,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T17:50:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:23,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125083507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:23,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/26779ff971904888ba58b6b2df407064 2024-11-20T17:50:23,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/c52543f2e4f5499d88fcd682ec9b2d0d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/c52543f2e4f5499d88fcd682ec9b2d0d 2024-11-20T17:50:23,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/c52543f2e4f5499d88fcd682ec9b2d0d, entries=200, sequenceid=338, filesize=14.4 K 2024-11-20T17:50:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/8ea930d11d2e42dfa45e643f266172ca as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8ea930d11d2e42dfa45e643f266172ca 2024-11-20T17:50:23,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8ea930d11d2e42dfa45e643f266172ca, entries=150, sequenceid=338, filesize=12.0 K 2024-11-20T17:50:23,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/26779ff971904888ba58b6b2df407064 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/26779ff971904888ba58b6b2df407064 2024-11-20T17:50:23,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/26779ff971904888ba58b6b2df407064, entries=150, sequenceid=338, filesize=12.0 K 2024-11-20T17:50:23,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0db4ba01acc0539d81a46ec1c40b3d92 in 861ms, sequenceid=338, compaction requested=true 2024-11-20T17:50:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:23,582 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:23,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:50:23,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:23,584 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:23,584 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:23,584 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,584 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/144307fba5d0410c882c964b33d78bf4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/bf0b9d8b262f426aa455b216560b5af7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/c52543f2e4f5499d88fcd682ec9b2d0d] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=39.2 K 2024-11-20T17:50:23,584 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:23,584 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:23,584 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,584 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1ef895a07f0348899fe7116984ef230f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78c83fa283f949be857df049f0c0194d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8ea930d11d2e42dfa45e643f266172ca] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.8 K 2024-11-20T17:50:23,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef895a07f0348899fe7116984ef230f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732125019820 2024-11-20T17:50:23,585 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 144307fba5d0410c882c964b33d78bf4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732125019820 2024-11-20T17:50:23,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c83fa283f949be857df049f0c0194d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732125020982 2024-11-20T17:50:23,585 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0b9d8b262f426aa455b216560b5af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732125020982 2024-11-20T17:50:23,585 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea930d11d2e42dfa45e643f266172ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732125022025 2024-11-20T17:50:23,585 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c52543f2e4f5499d88fcd682ec9b2d0d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732125022025 2024-11-20T17:50:23,592 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:23,592 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/78810f6f9ebb4fc083ac0f04a0fabfc9 is 50, key is test_row_0/B:col10/1732125022072/Put/seqid=0 2024-11-20T17:50:23,594 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:23,595 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/974852b3e1e34f0386c9399000899717 is 50, key is test_row_0/A:col10/1732125022072/Put/seqid=0 2024-11-20T17:50:23,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742392_1568 (size=13153) 2024-11-20T17:50:23,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742393_1569 (size=13153) 2024-11-20T17:50:23,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T17:50:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,612 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T17:50:23,612 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/974852b3e1e34f0386c9399000899717 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/974852b3e1e34f0386c9399000899717 2024-11-20T17:50:23,616 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into 974852b3e1e34f0386c9399000899717(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:23,616 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:23,616 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125023582; duration=0sec 2024-11-20T17:50:23,617 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:23,617 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:23,617 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7be0568f9f384491aa49ff61999a36b6 is 50, key is test_row_0/A:col10/1732125022775/Put/seqid=0 2024-11-20T17:50:23,619 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:23,619 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:23,619 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:23,620 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6a930c6c4a35425992a8dc9a96eb90c7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/684bc584119c41feb0f53a72895d7339, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/26779ff971904888ba58b6b2df407064] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.8 K 2024-11-20T17:50:23,620 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a930c6c4a35425992a8dc9a96eb90c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732125019820 2024-11-20T17:50:23,620 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 684bc584119c41feb0f53a72895d7339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732125020982 2024-11-20T17:50:23,621 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26779ff971904888ba58b6b2df407064, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732125022025 2024-11-20T17:50:23,626 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:23,627 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/b5f39dcd282946e59bcc90c193fcf735 is 50, key is test_row_0/C:col10/1732125022072/Put/seqid=0 2024-11-20T17:50:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T17:50:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742394_1570 (size=12301) 2024-11-20T17:50:23,635 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7be0568f9f384491aa49ff61999a36b6 2024-11-20T17:50:23,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/5cdd374abc8149a9a9c23291ac2f25e4 is 50, key is test_row_0/B:col10/1732125022775/Put/seqid=0 2024-11-20T17:50:23,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742395_1571 (size=13153) 2024-11-20T17:50:23,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742396_1572 (size=12301) 2024-11-20T17:50:24,012 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/78810f6f9ebb4fc083ac0f04a0fabfc9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78810f6f9ebb4fc083ac0f04a0fabfc9 2024-11-20T17:50:24,017 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into 78810f6f9ebb4fc083ac0f04a0fabfc9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:24,017 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:24,017 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125023582; duration=0sec 2024-11-20T17:50:24,017 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:24,017 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:24,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:24,051 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/5cdd374abc8149a9a9c23291ac2f25e4 2024-11-20T17:50:24,055 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/b5f39dcd282946e59bcc90c193fcf735 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b5f39dcd282946e59bcc90c193fcf735 2024-11-20T17:50:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9bdd5ced45d741e5a42510661a494935 is 50, key is test_row_0/C:col10/1732125022775/Put/seqid=0 2024-11-20T17:50:24,061 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into b5f39dcd282946e59bcc90c193fcf735(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:24,061 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:24,061 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125023583; duration=0sec 2024-11-20T17:50:24,061 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:24,061 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:24,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742397_1573 (size=12301) 2024-11-20T17:50:24,063 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9bdd5ced45d741e5a42510661a494935 2024-11-20T17:50:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7be0568f9f384491aa49ff61999a36b6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7be0568f9f384491aa49ff61999a36b6 2024-11-20T17:50:24,070 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7be0568f9f384491aa49ff61999a36b6, entries=150, sequenceid=361, filesize=12.0 K 2024-11-20T17:50:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/5cdd374abc8149a9a9c23291ac2f25e4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/5cdd374abc8149a9a9c23291ac2f25e4 2024-11-20T17:50:24,075 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/5cdd374abc8149a9a9c23291ac2f25e4, entries=150, sequenceid=361, filesize=12.0 K 2024-11-20T17:50:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/9bdd5ced45d741e5a42510661a494935 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9bdd5ced45d741e5a42510661a494935 2024-11-20T17:50:24,079 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9bdd5ced45d741e5a42510661a494935, entries=150, sequenceid=361, filesize=12.0 K 2024-11-20T17:50:24,080 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=60.38 KB/61830 for 0db4ba01acc0539d81a46ec1c40b3d92 in 468ms, sequenceid=361, compaction requested=false 2024-11-20T17:50:24,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:24,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-20T17:50:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-20T17:50:24,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T17:50:24,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-11-20T17:50:24,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.0850 sec 2024-11-20T17:50:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:24,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:24,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:50:24,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/08420ee35ede43b09b3c1cdfc9d59c9a is 50, key is test_row_0/A:col10/1732125024040/Put/seqid=0 2024-11-20T17:50:24,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742398_1574 (size=14741) 2024-11-20T17:50:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T17:50:24,138 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:24,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-20T17:50:24,139 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:24,140 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:24,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:24,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125084256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:24,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125084367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:24,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:24,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:24,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:24,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/08420ee35ede43b09b3c1cdfc9d59c9a 2024-11-20T17:50:24,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/d66d1650273c439d830b2b7fb7ac4041 is 50, key is test_row_0/B:col10/1732125024040/Put/seqid=0 2024-11-20T17:50:24,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742399_1575 (size=12301) 2024-11-20T17:50:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125084577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:24,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:24,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125084887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:24,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:24,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:24,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:24,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/d66d1650273c439d830b2b7fb7ac4041 2024-11-20T17:50:24,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/46409b8c75e84004b6c95c5953bed703 is 50, key is test_row_0/C:col10/1732125024040/Put/seqid=0 2024-11-20T17:50:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742400_1576 (size=12301) 2024-11-20T17:50:25,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60326 deadline: 1732125085167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,175 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:25,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60370 deadline: 1732125085187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60372 deadline: 1732125085187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,195 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,196 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60352 deadline: 1732125085207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,210 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in 0db4ba01acc0539d81a46ec1c40b3d92 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:25,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/46409b8c75e84004b6c95c5953bed703 2024-11-20T17:50:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/08420ee35ede43b09b3c1cdfc9d59c9a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/08420ee35ede43b09b3c1cdfc9d59c9a 2024-11-20T17:50:25,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/08420ee35ede43b09b3c1cdfc9d59c9a, entries=200, sequenceid=377, filesize=14.4 K 2024-11-20T17:50:25,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/d66d1650273c439d830b2b7fb7ac4041 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d66d1650273c439d830b2b7fb7ac4041 2024-11-20T17:50:25,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d66d1650273c439d830b2b7fb7ac4041, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T17:50:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/46409b8c75e84004b6c95c5953bed703 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/46409b8c75e84004b6c95c5953bed703 2024-11-20T17:50:25,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/46409b8c75e84004b6c95c5953bed703, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T17:50:25,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1320ms, sequenceid=377, compaction requested=true 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:25,406 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:25,406 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0db4ba01acc0539d81a46ec1c40b3d92:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:25,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:25,407 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:25,407 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/A is initiating minor compaction (all files) 2024-11-20T17:50:25,407 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/A in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,407 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/974852b3e1e34f0386c9399000899717, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7be0568f9f384491aa49ff61999a36b6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/08420ee35ede43b09b3c1cdfc9d59c9a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=39.3 K 2024-11-20T17:50:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:25,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:50:25,407 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 974852b3e1e34f0386c9399000899717, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732125022025 2024-11-20T17:50:25,408 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be0568f9f384491aa49ff61999a36b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732125022775 2024-11-20T17:50:25,408 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 08420ee35ede43b09b3c1cdfc9d59c9a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732125024040 2024-11-20T17:50:25,409 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:25,409 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/B is initiating minor compaction (all files) 2024-11-20T17:50:25,409 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/B in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,409 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78810f6f9ebb4fc083ac0f04a0fabfc9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/5cdd374abc8149a9a9c23291ac2f25e4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d66d1650273c439d830b2b7fb7ac4041] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.9 K 2024-11-20T17:50:25,410 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78810f6f9ebb4fc083ac0f04a0fabfc9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732125022025 2024-11-20T17:50:25,410 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cdd374abc8149a9a9c23291ac2f25e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732125022775 2024-11-20T17:50:25,410 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d66d1650273c439d830b2b7fb7ac4041, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732125024040 2024-11-20T17:50:25,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7c6827a3c504496a919242a4327a478d is 50, key is test_row_0/A:col10/1732125025406/Put/seqid=0 2024-11-20T17:50:25,426 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#A#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:25,427 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:25,427 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/a069102db795423d84dd88b63ec95ea3 is 50, key is test_row_0/A:col10/1732125024040/Put/seqid=0 2024-11-20T17:50:25,427 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/cd477d38db584a759e2fdcce46d3ecee is 50, key is test_row_0/B:col10/1732125024040/Put/seqid=0 2024-11-20T17:50:25,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742401_1577 (size=14741) 2024-11-20T17:50:25,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7c6827a3c504496a919242a4327a478d 2024-11-20T17:50:25,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742403_1579 (size=13255) 2024-11-20T17:50:25,441 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/a069102db795423d84dd88b63ec95ea3 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/a069102db795423d84dd88b63ec95ea3 2024-11-20T17:50:25,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/f6fa1a48cf714278b71259d585ca7fef is 50, key is test_row_0/B:col10/1732125025406/Put/seqid=0 2024-11-20T17:50:25,453 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/A of 0db4ba01acc0539d81a46ec1c40b3d92 into a069102db795423d84dd88b63ec95ea3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:25,453 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:25,453 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/A, priority=13, startTime=1732125025406; duration=0sec 2024-11-20T17:50:25,453 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:25,453 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:A 2024-11-20T17:50:25,454 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:25,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742402_1578 (size=13255) 2024-11-20T17:50:25,455 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:25,455 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 0db4ba01acc0539d81a46ec1c40b3d92/C is initiating minor compaction (all files) 2024-11-20T17:50:25,455 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0db4ba01acc0539d81a46ec1c40b3d92/C in TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,455 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b5f39dcd282946e59bcc90c193fcf735, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9bdd5ced45d741e5a42510661a494935, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/46409b8c75e84004b6c95c5953bed703] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp, totalSize=36.9 K 2024-11-20T17:50:25,455 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b5f39dcd282946e59bcc90c193fcf735, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732125022025 2024-11-20T17:50:25,455 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdd5ced45d741e5a42510661a494935, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732125022775 2024-11-20T17:50:25,455 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 46409b8c75e84004b6c95c5953bed703, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732125024040 2024-11-20T17:50:25,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742404_1580 (size=12301) 2024-11-20T17:50:25,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/f6fa1a48cf714278b71259d585ca7fef 2024-11-20T17:50:25,475 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0db4ba01acc0539d81a46ec1c40b3d92#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:25,476 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/bb06b6dde5c44af29d9c11e6e739c561 is 50, key is test_row_0/C:col10/1732125024040/Put/seqid=0 2024-11-20T17:50:25,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/ee41aeae6dfa4678a0e427e777709783 is 50, key is test_row_0/C:col10/1732125025406/Put/seqid=0 2024-11-20T17:50:25,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742405_1581 (size=13255) 2024-11-20T17:50:25,496 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/bb06b6dde5c44af29d9c11e6e739c561 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/bb06b6dde5c44af29d9c11e6e739c561 2024-11-20T17:50:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742406_1582 (size=12301) 2024-11-20T17:50:25,503 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/C of 0db4ba01acc0539d81a46ec1c40b3d92 into bb06b6dde5c44af29d9c11e6e739c561(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:25,503 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:25,503 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/C, priority=13, startTime=1732125025406; duration=0sec 2024-11-20T17:50:25,503 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:25,504 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:C 2024-11-20T17:50:25,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125085497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125085617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:25,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60320 deadline: 1732125085828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 , the old value is region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., hostname=613581fcdc40,44957,1732124872893, seqNum=2 because the exception is null or not the one we care about 2024-11-20T17:50:25,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:25,858 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/cd477d38db584a759e2fdcce46d3ecee as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/cd477d38db584a759e2fdcce46d3ecee 2024-11-20T17:50:25,863 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0db4ba01acc0539d81a46ec1c40b3d92/B of 0db4ba01acc0539d81a46ec1c40b3d92 into cd477d38db584a759e2fdcce46d3ecee(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:25,863 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:25,863 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., storeName=0db4ba01acc0539d81a46ec1c40b3d92/B, priority=13, startTime=1732125025406; duration=0sec 2024-11-20T17:50:25,863 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:25,863 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0db4ba01acc0539d81a46ec1c40b3d92:B 2024-11-20T17:50:25,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/ee41aeae6dfa4678a0e427e777709783 2024-11-20T17:50:25,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/7c6827a3c504496a919242a4327a478d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7c6827a3c504496a919242a4327a478d 2024-11-20T17:50:25,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7c6827a3c504496a919242a4327a478d, entries=200, sequenceid=401, filesize=14.4 K 2024-11-20T17:50:25,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/f6fa1a48cf714278b71259d585ca7fef as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/f6fa1a48cf714278b71259d585ca7fef 2024-11-20T17:50:25,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/f6fa1a48cf714278b71259d585ca7fef, entries=150, sequenceid=401, filesize=12.0 K 2024-11-20T17:50:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/ee41aeae6dfa4678a0e427e777709783 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/ee41aeae6dfa4678a0e427e777709783 2024-11-20T17:50:25,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/ee41aeae6dfa4678a0e427e777709783, entries=150, sequenceid=401, filesize=12.0 K 2024-11-20T17:50:25,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0db4ba01acc0539d81a46ec1c40b3d92 in 511ms, sequenceid=401, compaction requested=false 2024-11-20T17:50:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:25,920 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-20T17:50:25,920 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:25,920 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,920 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:25,920 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-20T17:50:25,920 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:25,920 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,921 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:25,923 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-20T17:50:25,923 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:25,923 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,923 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:25,925 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-11-20T17:50:25,925 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:25,925 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,925 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:25,926 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-11-20T17:50:25,926 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:25,926 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,926 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:25,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:25,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T17:50:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:25,999 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:50:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/d9a0aee5714648ea841d1544c9051aad is 50, key is test_row_0/A:col10/1732125025471/Put/seqid=0 2024-11-20T17:50:26,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742407_1583 (size=12301) 2024-11-20T17:50:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:26,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. as already flushing 2024-11-20T17:50:26,139 INFO [Thread-2201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2201. 2024-11-20T17:50:26,139 DEBUG [Thread-2201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:26,139 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:26,139 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:26,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:26,415 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/d9a0aee5714648ea841d1544c9051aad 2024-11-20T17:50:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/4a3e7ed737e048a2ba11152b050c05bc is 50, key is test_row_0/B:col10/1732125025471/Put/seqid=0 2024-11-20T17:50:26,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742408_1584 (size=12301) 2024-11-20T17:50:26,830 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/4a3e7ed737e048a2ba11152b050c05bc 2024-11-20T17:50:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/e3120dd034fa4c5ea3fa547e8f64a115 is 50, key is test_row_0/C:col10/1732125025471/Put/seqid=0 2024-11-20T17:50:26,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742409_1585 (size=12301) 2024-11-20T17:50:27,238 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/e3120dd034fa4c5ea3fa547e8f64a115 2024-11-20T17:50:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/d9a0aee5714648ea841d1544c9051aad as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/d9a0aee5714648ea841d1544c9051aad 2024-11-20T17:50:27,244 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/d9a0aee5714648ea841d1544c9051aad, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T17:50:27,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/4a3e7ed737e048a2ba11152b050c05bc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/4a3e7ed737e048a2ba11152b050c05bc 2024-11-20T17:50:27,248 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/4a3e7ed737e048a2ba11152b050c05bc, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T17:50:27,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/e3120dd034fa4c5ea3fa547e8f64a115 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e3120dd034fa4c5ea3fa547e8f64a115 2024-11-20T17:50:27,252 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e3120dd034fa4c5ea3fa547e8f64a115, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T17:50:27,253 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=6.71 KB/6870 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1254ms, sequenceid=416, compaction requested=true 2024-11-20T17:50:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 0db4ba01acc0539d81a46ec1c40b3d92: 2024-11-20T17:50:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:27,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-20T17:50:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-20T17:50:27,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-20T17:50:27,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1140 sec 2024-11-20T17:50:27,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 3.1180 sec 2024-11-20T17:50:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T17:50:28,297 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:29,208 INFO [Thread-2203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2203. 2024-11-20T17:50:29,208 DEBUG [Thread-2203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:29,208 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,208 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:29,219 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-20T17:50:29,219 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:29,219 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,219 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:29,229 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-20T17:50:29,229 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:29,229 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,229 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:29,239 INFO [Thread-2199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2199. 2024-11-20T17:50:29,239 DEBUG [Thread-2199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:29,239 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,239 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:29,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T17:50:29,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T17:50:29,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-20T17:50:29,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-20T17:50:29,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1473 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4415 rows 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1474 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4421 rows 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1475 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4417 rows 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1476 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4427 rows 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1469 2024-11-20T17:50:29,240 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4406 rows 2024-11-20T17:50:29,240 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:50:29,240 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:50:29,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:29,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:29,241 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T17:50:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T17:50:29,245 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125029245"}]},"ts":"1732125029245"} 2024-11-20T17:50:29,247 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T17:50:29,247 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T17:50:29,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:50:29,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, UNASSIGN}] 2024-11-20T17:50:29,250 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, UNASSIGN 2024-11-20T17:50:29,251 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=0db4ba01acc0539d81a46ec1c40b3d92, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:29,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, UNASSIGN because future has completed 2024-11-20T17:50:29,252 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:50:29,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:50:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T17:50:29,404 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:29,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:50:29,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 0db4ba01acc0539d81a46ec1c40b3d92, disabling compactions & flushes 2024-11-20T17:50:29,404 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:29,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:29,404 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. after waiting 0 ms 2024-11-20T17:50:29,405 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:29,405 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 0db4ba01acc0539d81a46ec1c40b3d92 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T17:50:29,409 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/134b481435314179ac23a359b9535a50 is 50, key is test_row_0/A:col10/1732125029218/Put/seqid=0 2024-11-20T17:50:29,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742410_1586 (size=12301) 2024-11-20T17:50:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T17:50:29,812 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/134b481435314179ac23a359b9535a50 2024-11-20T17:50:29,820 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2125c4c645cf4feab322f9aa20493aee is 50, key is test_row_0/B:col10/1732125029218/Put/seqid=0 2024-11-20T17:50:29,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742411_1587 (size=12301) 2024-11-20T17:50:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T17:50:30,223 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2125c4c645cf4feab322f9aa20493aee 2024-11-20T17:50:30,230 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/c9e6329d407d4e6c972ca0f240669354 is 50, key is test_row_0/C:col10/1732125029218/Put/seqid=0 2024-11-20T17:50:30,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742412_1588 (size=12301) 2024-11-20T17:50:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T17:50:30,634 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/c9e6329d407d4e6c972ca0f240669354 2024-11-20T17:50:30,638 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/A/134b481435314179ac23a359b9535a50 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/134b481435314179ac23a359b9535a50 2024-11-20T17:50:30,642 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/134b481435314179ac23a359b9535a50, entries=150, sequenceid=424, filesize=12.0 K 2024-11-20T17:50:30,643 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/B/2125c4c645cf4feab322f9aa20493aee as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2125c4c645cf4feab322f9aa20493aee 2024-11-20T17:50:30,646 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2125c4c645cf4feab322f9aa20493aee, entries=150, sequenceid=424, filesize=12.0 K 2024-11-20T17:50:30,646 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/.tmp/C/c9e6329d407d4e6c972ca0f240669354 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/c9e6329d407d4e6c972ca0f240669354 2024-11-20T17:50:30,649 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/c9e6329d407d4e6c972ca0f240669354, entries=150, sequenceid=424, filesize=12.0 K 2024-11-20T17:50:30,650 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1245ms, sequenceid=424, compaction requested=true 2024-11-20T17:50:30,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/73d90c44905149e19023fa9a377ae007, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5b00ff2e33904a1c8bc1319cf3bb45c7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/118453d303eb4ce792d3edca20aa76de, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0447ab6f092441a9bbbd5aa83f681e86, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0fd373c088f4485597358392d230223a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f30c740e0b094e4b9d44892649c8981e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0d978066a33c4dacafe2531701d38e3e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f532a0bd2f674a5babf3a4a894c1c9cd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/1c6eea99c4be455d98edc8840784011e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/51a44a1c1a0f41c782f97507147121cb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cd9a3a1be4af4294b84bc5a1465a9e33, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/713a47f786c14350a335c9655ff38684, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/20bf01131cab4fda917388392090b880, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/66a853671f924ab785ae768f8784ad8e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/fade9ea3d56541e09bd4bf47c086556f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7bdbe31898064e1aa44110a04c7d9926, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5fd66b8037774b459d0db4484e2662db, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/9a736130ae4a4c478e692faf2263131e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/17bcdb2f68f642e59570cfbfec37da31, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cf74a909f1e6446abc4aef20bcfa2da0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/65faa5993d5d44a3b930941776afdb2e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0b63c8cebe8a4fec8619f9603604a6b1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/144307fba5d0410c882c964b33d78bf4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/bf0b9d8b262f426aa455b216560b5af7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/c52543f2e4f5499d88fcd682ec9b2d0d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/974852b3e1e34f0386c9399000899717, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7be0568f9f384491aa49ff61999a36b6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/08420ee35ede43b09b3c1cdfc9d59c9a] to archive 2024-11-20T17:50:30,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:50:30,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/73d90c44905149e19023fa9a377ae007 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/73d90c44905149e19023fa9a377ae007 2024-11-20T17:50:30,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5b00ff2e33904a1c8bc1319cf3bb45c7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5b00ff2e33904a1c8bc1319cf3bb45c7 2024-11-20T17:50:30,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/118453d303eb4ce792d3edca20aa76de to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/118453d303eb4ce792d3edca20aa76de 2024-11-20T17:50:30,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0447ab6f092441a9bbbd5aa83f681e86 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0447ab6f092441a9bbbd5aa83f681e86 2024-11-20T17:50:30,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0fd373c088f4485597358392d230223a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0fd373c088f4485597358392d230223a 2024-11-20T17:50:30,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f30c740e0b094e4b9d44892649c8981e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f30c740e0b094e4b9d44892649c8981e 2024-11-20T17:50:30,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0d978066a33c4dacafe2531701d38e3e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0d978066a33c4dacafe2531701d38e3e 2024-11-20T17:50:30,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f532a0bd2f674a5babf3a4a894c1c9cd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/f532a0bd2f674a5babf3a4a894c1c9cd 2024-11-20T17:50:30,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/1c6eea99c4be455d98edc8840784011e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/1c6eea99c4be455d98edc8840784011e 2024-11-20T17:50:30,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/51a44a1c1a0f41c782f97507147121cb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/51a44a1c1a0f41c782f97507147121cb 2024-11-20T17:50:30,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cd9a3a1be4af4294b84bc5a1465a9e33 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cd9a3a1be4af4294b84bc5a1465a9e33 2024-11-20T17:50:30,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/713a47f786c14350a335c9655ff38684 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/713a47f786c14350a335c9655ff38684 2024-11-20T17:50:30,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/20bf01131cab4fda917388392090b880 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/20bf01131cab4fda917388392090b880 2024-11-20T17:50:30,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/66a853671f924ab785ae768f8784ad8e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/66a853671f924ab785ae768f8784ad8e 2024-11-20T17:50:30,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/fade9ea3d56541e09bd4bf47c086556f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/fade9ea3d56541e09bd4bf47c086556f 2024-11-20T17:50:30,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7bdbe31898064e1aa44110a04c7d9926 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7bdbe31898064e1aa44110a04c7d9926 2024-11-20T17:50:30,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5fd66b8037774b459d0db4484e2662db to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/5fd66b8037774b459d0db4484e2662db 2024-11-20T17:50:30,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/9a736130ae4a4c478e692faf2263131e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/9a736130ae4a4c478e692faf2263131e 2024-11-20T17:50:30,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/17bcdb2f68f642e59570cfbfec37da31 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/17bcdb2f68f642e59570cfbfec37da31 2024-11-20T17:50:30,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cf74a909f1e6446abc4aef20bcfa2da0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/cf74a909f1e6446abc4aef20bcfa2da0 2024-11-20T17:50:30,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/65faa5993d5d44a3b930941776afdb2e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/65faa5993d5d44a3b930941776afdb2e 2024-11-20T17:50:30,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0b63c8cebe8a4fec8619f9603604a6b1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/0b63c8cebe8a4fec8619f9603604a6b1 2024-11-20T17:50:30,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/144307fba5d0410c882c964b33d78bf4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/144307fba5d0410c882c964b33d78bf4 2024-11-20T17:50:30,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/bf0b9d8b262f426aa455b216560b5af7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/bf0b9d8b262f426aa455b216560b5af7 2024-11-20T17:50:30,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/c52543f2e4f5499d88fcd682ec9b2d0d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/c52543f2e4f5499d88fcd682ec9b2d0d 2024-11-20T17:50:30,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/974852b3e1e34f0386c9399000899717 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/974852b3e1e34f0386c9399000899717 2024-11-20T17:50:30,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7be0568f9f384491aa49ff61999a36b6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7be0568f9f384491aa49ff61999a36b6 2024-11-20T17:50:30,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/08420ee35ede43b09b3c1cdfc9d59c9a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/08420ee35ede43b09b3c1cdfc9d59c9a 2024-11-20T17:50:30,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/ff073f401b384a3a960f1eb02894a431, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e5b2b87f83004411936ab69d5475e56f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a566111f565646b0a842aa399adb87e4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2f8736fb51654aedad18426efd8a1772, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e10e275f65314b5a8ddfe3736565b6f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d661b3da3e914382b159faea2a340487, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8d1e632a8b794694a6c52c4e71841458, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/69e791c5b5984f65a76905d13c1341d0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9036b8c43ec9402093944e687d3dac77, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a529d153da874b958756525747443333, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/c55f884d98484de2a0ff30881d91fc62, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b43fd927da214421a05fb32188e7fb95, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6ffa161873ce4f38841dfa314bfc1b73, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2749c4ed726245e59456acbf7ae9a008, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6a6b4adff21f435b8fb239bef7b6bda7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9e1175f6f01948f7b1658f4b47809c29, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1596ee8897024d6e9f3d7447f9717894, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8104755200b7480581ebe6f103e5d6c3, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b8b9e970c7ac457e9bb9da5e9e30468f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2b527fe7b4c3460ea1b900f192a0b3d9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/7d0903a974b44465ab3b62ef4ba04800, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1ef895a07f0348899fe7116984ef230f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/bcba3cc755c3404b9ce558ee7c5c8562, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78c83fa283f949be857df049f0c0194d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78810f6f9ebb4fc083ac0f04a0fabfc9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8ea930d11d2e42dfa45e643f266172ca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/5cdd374abc8149a9a9c23291ac2f25e4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d66d1650273c439d830b2b7fb7ac4041] to archive 2024-11-20T17:50:30,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:50:30,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/ff073f401b384a3a960f1eb02894a431 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/ff073f401b384a3a960f1eb02894a431 2024-11-20T17:50:30,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e5b2b87f83004411936ab69d5475e56f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e5b2b87f83004411936ab69d5475e56f 2024-11-20T17:50:30,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a566111f565646b0a842aa399adb87e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a566111f565646b0a842aa399adb87e4 2024-11-20T17:50:30,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2f8736fb51654aedad18426efd8a1772 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2f8736fb51654aedad18426efd8a1772 2024-11-20T17:50:30,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e10e275f65314b5a8ddfe3736565b6f9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/e10e275f65314b5a8ddfe3736565b6f9 2024-11-20T17:50:30,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d661b3da3e914382b159faea2a340487 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d661b3da3e914382b159faea2a340487 2024-11-20T17:50:30,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8d1e632a8b794694a6c52c4e71841458 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8d1e632a8b794694a6c52c4e71841458 2024-11-20T17:50:30,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/69e791c5b5984f65a76905d13c1341d0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/69e791c5b5984f65a76905d13c1341d0 2024-11-20T17:50:30,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9036b8c43ec9402093944e687d3dac77 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9036b8c43ec9402093944e687d3dac77 2024-11-20T17:50:30,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a529d153da874b958756525747443333 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/a529d153da874b958756525747443333 2024-11-20T17:50:30,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/c55f884d98484de2a0ff30881d91fc62 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/c55f884d98484de2a0ff30881d91fc62 2024-11-20T17:50:30,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b43fd927da214421a05fb32188e7fb95 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b43fd927da214421a05fb32188e7fb95 2024-11-20T17:50:30,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6ffa161873ce4f38841dfa314bfc1b73 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6ffa161873ce4f38841dfa314bfc1b73 2024-11-20T17:50:30,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2749c4ed726245e59456acbf7ae9a008 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2749c4ed726245e59456acbf7ae9a008 2024-11-20T17:50:30,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6a6b4adff21f435b8fb239bef7b6bda7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/6a6b4adff21f435b8fb239bef7b6bda7 2024-11-20T17:50:30,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9e1175f6f01948f7b1658f4b47809c29 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/9e1175f6f01948f7b1658f4b47809c29 2024-11-20T17:50:30,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1596ee8897024d6e9f3d7447f9717894 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1596ee8897024d6e9f3d7447f9717894 2024-11-20T17:50:30,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8104755200b7480581ebe6f103e5d6c3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8104755200b7480581ebe6f103e5d6c3 2024-11-20T17:50:30,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b8b9e970c7ac457e9bb9da5e9e30468f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/b8b9e970c7ac457e9bb9da5e9e30468f 2024-11-20T17:50:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2b527fe7b4c3460ea1b900f192a0b3d9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2b527fe7b4c3460ea1b900f192a0b3d9 2024-11-20T17:50:30,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/7d0903a974b44465ab3b62ef4ba04800 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/7d0903a974b44465ab3b62ef4ba04800 2024-11-20T17:50:30,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1ef895a07f0348899fe7116984ef230f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/1ef895a07f0348899fe7116984ef230f 2024-11-20T17:50:30,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/bcba3cc755c3404b9ce558ee7c5c8562 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/bcba3cc755c3404b9ce558ee7c5c8562 2024-11-20T17:50:30,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78c83fa283f949be857df049f0c0194d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78c83fa283f949be857df049f0c0194d 2024-11-20T17:50:30,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78810f6f9ebb4fc083ac0f04a0fabfc9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/78810f6f9ebb4fc083ac0f04a0fabfc9 2024-11-20T17:50:30,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8ea930d11d2e42dfa45e643f266172ca to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/8ea930d11d2e42dfa45e643f266172ca 2024-11-20T17:50:30,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/5cdd374abc8149a9a9c23291ac2f25e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/5cdd374abc8149a9a9c23291ac2f25e4 2024-11-20T17:50:30,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d66d1650273c439d830b2b7fb7ac4041 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/d66d1650273c439d830b2b7fb7ac4041 2024-11-20T17:50:30,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/45f23935856f4ca0b69f43eb9bea47d4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/8d307f7e6f424ab688145efa4444bdce, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9f2473ea2d5f4318975959812678829a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/64512161c4f943a495354612c35b6501, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/21c59d59dd5e4c0193c63b5cb8d47cb7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c89bbb215ab4f1db8f6dae32b742c6c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/2e628440ffe444f78c98781f7fd35b87, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/a77b4a2edd894f56b4d29526866711ae, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c825b613ce441c8883819ef4aa19a89, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/59a4a9f8391d4804bc2ca387a53a1564, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/71dce67c4bf0492aa488b67790e0f91e, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/63630a72efea414b803079645a9100dc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9791bdee6d9c4a14b3e034547c98f77d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f887e0372b5743d586342f6fd1e5c58f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f7670b1dd7b64bb0ba662d8a729ab2e4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6b53ef1a1c1b45588f02338afd7fc60d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/65707b2ee2f54bc296c383203eb3ebd4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/734572c3f6e2416cbdfecadd04cb8490, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e891ccfe55234f3087e27acba13c7793, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/745d9e83f4104b2e9a4edc22851e8b85, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b1023ee4d73046229c1f8771d8972ea0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6a930c6c4a35425992a8dc9a96eb90c7, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/cac25bf0b3a348b890302e3a7bf4fc6a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/684bc584119c41feb0f53a72895d7339, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b5f39dcd282946e59bcc90c193fcf735, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/26779ff971904888ba58b6b2df407064, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9bdd5ced45d741e5a42510661a494935, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/46409b8c75e84004b6c95c5953bed703] to archive 2024-11-20T17:50:30,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:50:30,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/45f23935856f4ca0b69f43eb9bea47d4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/45f23935856f4ca0b69f43eb9bea47d4 2024-11-20T17:50:30,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/8d307f7e6f424ab688145efa4444bdce to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/8d307f7e6f424ab688145efa4444bdce 2024-11-20T17:50:30,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9f2473ea2d5f4318975959812678829a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9f2473ea2d5f4318975959812678829a 2024-11-20T17:50:30,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/64512161c4f943a495354612c35b6501 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/64512161c4f943a495354612c35b6501 2024-11-20T17:50:30,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/21c59d59dd5e4c0193c63b5cb8d47cb7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/21c59d59dd5e4c0193c63b5cb8d47cb7 2024-11-20T17:50:30,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c89bbb215ab4f1db8f6dae32b742c6c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c89bbb215ab4f1db8f6dae32b742c6c 2024-11-20T17:50:30,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/2e628440ffe444f78c98781f7fd35b87 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/2e628440ffe444f78c98781f7fd35b87 2024-11-20T17:50:30,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/a77b4a2edd894f56b4d29526866711ae to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/a77b4a2edd894f56b4d29526866711ae 2024-11-20T17:50:30,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c825b613ce441c8883819ef4aa19a89 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/0c825b613ce441c8883819ef4aa19a89 2024-11-20T17:50:30,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/59a4a9f8391d4804bc2ca387a53a1564 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/59a4a9f8391d4804bc2ca387a53a1564 2024-11-20T17:50:30,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/71dce67c4bf0492aa488b67790e0f91e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/71dce67c4bf0492aa488b67790e0f91e 2024-11-20T17:50:30,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/63630a72efea414b803079645a9100dc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/63630a72efea414b803079645a9100dc 2024-11-20T17:50:30,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9791bdee6d9c4a14b3e034547c98f77d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9791bdee6d9c4a14b3e034547c98f77d 2024-11-20T17:50:30,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f887e0372b5743d586342f6fd1e5c58f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f887e0372b5743d586342f6fd1e5c58f 2024-11-20T17:50:30,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f7670b1dd7b64bb0ba662d8a729ab2e4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/f7670b1dd7b64bb0ba662d8a729ab2e4 2024-11-20T17:50:30,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6b53ef1a1c1b45588f02338afd7fc60d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6b53ef1a1c1b45588f02338afd7fc60d 2024-11-20T17:50:30,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/65707b2ee2f54bc296c383203eb3ebd4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/65707b2ee2f54bc296c383203eb3ebd4 2024-11-20T17:50:30,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/734572c3f6e2416cbdfecadd04cb8490 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/734572c3f6e2416cbdfecadd04cb8490 2024-11-20T17:50:30,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e891ccfe55234f3087e27acba13c7793 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e891ccfe55234f3087e27acba13c7793 2024-11-20T17:50:30,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/745d9e83f4104b2e9a4edc22851e8b85 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/745d9e83f4104b2e9a4edc22851e8b85 2024-11-20T17:50:30,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b1023ee4d73046229c1f8771d8972ea0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b1023ee4d73046229c1f8771d8972ea0 2024-11-20T17:50:30,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6a930c6c4a35425992a8dc9a96eb90c7 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/6a930c6c4a35425992a8dc9a96eb90c7 2024-11-20T17:50:30,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/cac25bf0b3a348b890302e3a7bf4fc6a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/cac25bf0b3a348b890302e3a7bf4fc6a 2024-11-20T17:50:30,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/684bc584119c41feb0f53a72895d7339 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/684bc584119c41feb0f53a72895d7339 2024-11-20T17:50:30,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b5f39dcd282946e59bcc90c193fcf735 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/b5f39dcd282946e59bcc90c193fcf735 2024-11-20T17:50:30,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/26779ff971904888ba58b6b2df407064 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/26779ff971904888ba58b6b2df407064 2024-11-20T17:50:30,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9bdd5ced45d741e5a42510661a494935 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/9bdd5ced45d741e5a42510661a494935 2024-11-20T17:50:30,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/46409b8c75e84004b6c95c5953bed703 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/46409b8c75e84004b6c95c5953bed703 2024-11-20T17:50:30,738 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/recovered.edits/427.seqid, newMaxSeqId=427, maxSeqId=1 2024-11-20T17:50:30,738 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. 2024-11-20T17:50:30,738 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 0db4ba01acc0539d81a46ec1c40b3d92: Waiting for close lock at 1732125029404Running coprocessor pre-close hooks at 1732125029404Disabling compacts and flushes for region at 1732125029404Disabling writes for close at 1732125029404Obtaining lock to block concurrent updates at 1732125029405 (+1 ms)Preparing flush snapshotting stores in 0db4ba01acc0539d81a46ec1c40b3d92 at 1732125029405Finished memstore snapshotting TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732125029405Flushing stores of TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92. at 1732125029406 (+1 ms)Flushing 0db4ba01acc0539d81a46ec1c40b3d92/A: creating writer at 1732125029406Flushing 0db4ba01acc0539d81a46ec1c40b3d92/A: appending metadata at 1732125029408 (+2 ms)Flushing 0db4ba01acc0539d81a46ec1c40b3d92/A: closing flushed file at 1732125029408Flushing 0db4ba01acc0539d81a46ec1c40b3d92/B: creating writer at 1732125029816 (+408 ms)Flushing 0db4ba01acc0539d81a46ec1c40b3d92/B: appending metadata at 1732125029819 (+3 ms)Flushing 0db4ba01acc0539d81a46ec1c40b3d92/B: closing flushed file at 1732125029819Flushing 0db4ba01acc0539d81a46ec1c40b3d92/C: creating writer at 1732125030228 (+409 ms)Flushing 0db4ba01acc0539d81a46ec1c40b3d92/C: appending metadata at 1732125030230 (+2 ms)Flushing 0db4ba01acc0539d81a46ec1c40b3d92/C: closing flushed file at 1732125030230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc114d7: reopening flushed file at 1732125030638 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52373097: reopening flushed file at 1732125030643 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40011ef5: reopening flushed file at 1732125030646 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0db4ba01acc0539d81a46ec1c40b3d92 in 1245ms, sequenceid=424, compaction requested=true at 1732125030650 (+4 ms)Writing region close event to WAL at 1732125030735 (+85 ms)Running coprocessor post-close hooks at 1732125030738 (+3 ms)Closed at 1732125030738 2024-11-20T17:50:30,740 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:30,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=0db4ba01acc0539d81a46ec1c40b3d92, regionState=CLOSED 2024-11-20T17:50:30,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:50:30,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-20T17:50:30,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 0db4ba01acc0539d81a46ec1c40b3d92, server=613581fcdc40,44957,1732124872893 in 1.4900 sec 2024-11-20T17:50:30,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-20T17:50:30,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0db4ba01acc0539d81a46ec1c40b3d92, UNASSIGN in 1.4950 sec 2024-11-20T17:50:30,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-20T17:50:30,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4980 sec 2024-11-20T17:50:30,748 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125030748"}]},"ts":"1732125030748"} 2024-11-20T17:50:30,749 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T17:50:30,749 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T17:50:30,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5090 sec 2024-11-20T17:50:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T17:50:31,387 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:31,388 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T17:50:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,390 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T17:50:31,391 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:31,393 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/recovered.edits] 2024-11-20T17:50:31,395 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/134b481435314179ac23a359b9535a50 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/134b481435314179ac23a359b9535a50 2024-11-20T17:50:31,396 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7c6827a3c504496a919242a4327a478d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/7c6827a3c504496a919242a4327a478d 2024-11-20T17:50:31,397 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/a069102db795423d84dd88b63ec95ea3 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/a069102db795423d84dd88b63ec95ea3 2024-11-20T17:50:31,398 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/d9a0aee5714648ea841d1544c9051aad to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/A/d9a0aee5714648ea841d1544c9051aad 2024-11-20T17:50:31,399 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2125c4c645cf4feab322f9aa20493aee to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/2125c4c645cf4feab322f9aa20493aee 2024-11-20T17:50:31,400 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/4a3e7ed737e048a2ba11152b050c05bc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/4a3e7ed737e048a2ba11152b050c05bc 2024-11-20T17:50:31,401 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/cd477d38db584a759e2fdcce46d3ecee to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/cd477d38db584a759e2fdcce46d3ecee 2024-11-20T17:50:31,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/f6fa1a48cf714278b71259d585ca7fef to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/B/f6fa1a48cf714278b71259d585ca7fef 2024-11-20T17:50:31,403 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/bb06b6dde5c44af29d9c11e6e739c561 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/bb06b6dde5c44af29d9c11e6e739c561 2024-11-20T17:50:31,404 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/c9e6329d407d4e6c972ca0f240669354 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/c9e6329d407d4e6c972ca0f240669354 2024-11-20T17:50:31,405 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e3120dd034fa4c5ea3fa547e8f64a115 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/e3120dd034fa4c5ea3fa547e8f64a115 2024-11-20T17:50:31,405 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/ee41aeae6dfa4678a0e427e777709783 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/C/ee41aeae6dfa4678a0e427e777709783 2024-11-20T17:50:31,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/recovered.edits/427.seqid to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92/recovered.edits/427.seqid 2024-11-20T17:50:31,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/0db4ba01acc0539d81a46ec1c40b3d92 2024-11-20T17:50:31,408 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T17:50:31,410 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,411 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T17:50:31,413 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T17:50:31,414 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,414 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T17:50:31,414 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732125031414"}]},"ts":"9223372036854775807"} 2024-11-20T17:50:31,416 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T17:50:31,416 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0db4ba01acc0539d81a46ec1c40b3d92, NAME => 'TestAcidGuarantees,,1732125004736.0db4ba01acc0539d81a46ec1c40b3d92.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T17:50:31,417 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T17:50:31,417 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732125031417"}]},"ts":"9223372036854775807"} 2024-11-20T17:50:31,419 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T17:50:31,419 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-20T17:50:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T17:50:31,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T17:50:31,497 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:31,506 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 242), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=611 (was 562) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=8389 (was 8185) - AvailableMemoryMB LEAK? - 2024-11-20T17:50:31,517 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=611, ProcessCount=11, AvailableMemoryMB=8389 2024-11-20T17:50:31,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:50:31,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:50:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:31,520 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T17:50:31,520 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:31,520 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-20T17:50:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T17:50:31,521 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T17:50:31,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742413_1589 (size=959) 2024-11-20T17:50:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T17:50:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T17:50:31,927 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7 2024-11-20T17:50:31,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742414_1590 (size=53) 2024-11-20T17:50:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T17:50:32,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:50:32,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 13207a781081472a057b14b506529437, disabling compactions & flushes 2024-11-20T17:50:32,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. after waiting 0 ms 2024-11-20T17:50:32,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13207a781081472a057b14b506529437: Waiting for close lock at 1732125032332Disabling compacts and flushes for region at 1732125032332Disabling writes for close at 1732125032332Writing region close event to WAL at 1732125032332Closed at 1732125032332 2024-11-20T17:50:32,333 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T17:50:32,333 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732125032333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125032333"}]},"ts":"1732125032333"} 2024-11-20T17:50:32,334 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T17:50:32,335 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T17:50:32,335 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125032335"}]},"ts":"1732125032335"} 2024-11-20T17:50:32,336 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T17:50:32,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, ASSIGN}] 2024-11-20T17:50:32,337 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, ASSIGN 2024-11-20T17:50:32,338 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, ASSIGN; state=OFFLINE, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=false 2024-11-20T17:50:32,488 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:32,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, ASSIGN because future has completed 2024-11-20T17:50:32,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:50:32,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T17:50:32,644 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,644 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:50:32,644 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 13207a781081472a057b14b506529437 2024-11-20T17:50:32,644 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:50:32,644 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,644 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,645 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 13207a781081472a057b14b506529437 2024-11-20T17:50:32,647 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13207a781081472a057b14b506529437 columnFamilyName A 2024-11-20T17:50:32,647 DEBUG [StoreOpener-13207a781081472a057b14b506529437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:32,647 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(327): Store=13207a781081472a057b14b506529437/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:32,647 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 13207a781081472a057b14b506529437 2024-11-20T17:50:32,648 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13207a781081472a057b14b506529437 columnFamilyName B 2024-11-20T17:50:32,648 DEBUG [StoreOpener-13207a781081472a057b14b506529437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:32,648 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(327): Store=13207a781081472a057b14b506529437/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:32,648 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 13207a781081472a057b14b506529437 2024-11-20T17:50:32,649 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13207a781081472a057b14b506529437 columnFamilyName C 2024-11-20T17:50:32,650 DEBUG [StoreOpener-13207a781081472a057b14b506529437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:32,650 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(327): Store=13207a781081472a057b14b506529437/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:32,650 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,650 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437 2024-11-20T17:50:32,651 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437 2024-11-20T17:50:32,651 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,651 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,652 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:50:32,653 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,654 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:50:32,655 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened 13207a781081472a057b14b506529437; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69839184, jitterRate=0.04068493843078613}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:50:32,655 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13207a781081472a057b14b506529437 2024-11-20T17:50:32,655 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for 13207a781081472a057b14b506529437: Running coprocessor pre-open hook at 1732125032644Writing region info on filesystem at 1732125032644Initializing all the Stores at 1732125032645 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125032645Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125032645Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125032645Cleaning up temporary data from old regions at 1732125032651 (+6 ms)Running coprocessor post-open hooks at 1732125032655 (+4 ms)Region opened successfully at 1732125032655 2024-11-20T17:50:32,656 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., pid=150, masterSystemTime=1732125032641 2024-11-20T17:50:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T17:50:32,658 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,658 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:32,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:32,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:50:32,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-20T17:50:32,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 in 170 msec 2024-11-20T17:50:32,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T17:50:32,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, ASSIGN in 325 msec 2024-11-20T17:50:32,663 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T17:50:32,663 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125032663"}]},"ts":"1732125032663"} 2024-11-20T17:50:32,664 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T17:50:32,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T17:50:32,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-20T17:50:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T17:50:33,667 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:33,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d903c35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:33,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:33,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:33,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9f803e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:33,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797f7c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:33,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:33,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:33,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T17:50:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,39775,1732124872132 2024-11-20T17:50:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d8cb563 2024-11-20T17:50:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T17:50:33,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T17:50:33,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T17:50:33,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:50:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T17:50:33,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742415_1591 (size=995) 2024-11-20T17:50:34,089 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T17:50:34,089 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T17:50:34,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:50:34,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, REOPEN/MOVE}] 2024-11-20T17:50:34,094 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, REOPEN/MOVE 2024-11-20T17:50:34,094 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, REOPEN/MOVE because future has completed 2024-11-20T17:50:34,096 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:50:34,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:50:34,248 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 13207a781081472a057b14b506529437 2024-11-20T17:50:34,249 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:50:34,249 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 13207a781081472a057b14b506529437, disabling compactions & flushes 2024-11-20T17:50:34,249 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,249 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,249 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. after waiting 0 ms 2024-11-20T17:50:34,249 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,252 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T17:50:34,253 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,253 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 13207a781081472a057b14b506529437: Waiting for close lock at 1732125034249Running coprocessor pre-close hooks at 1732125034249Disabling compacts and flushes for region at 1732125034249Disabling writes for close at 1732125034249Writing region close event to WAL at 1732125034249Running coprocessor post-close hooks at 1732125034253 (+4 ms)Closed at 1732125034253 2024-11-20T17:50:34,253 WARN [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: 13207a781081472a057b14b506529437 to self. 2024-11-20T17:50:34,255 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 13207a781081472a057b14b506529437 2024-11-20T17:50:34,255 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=CLOSED 2024-11-20T17:50:34,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:50:34,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-20T17:50:34,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 in 161 msec 2024-11-20T17:50:34,260 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, REOPEN/MOVE; state=CLOSED, location=613581fcdc40,44957,1732124872893; forceNewPlan=false, retain=true 2024-11-20T17:50:34,411 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=OPENING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, REOPEN/MOVE because future has completed 2024-11-20T17:50:34,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:50:34,567 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,567 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:50:34,567 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 13207a781081472a057b14b506529437 2024-11-20T17:50:34,567 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:50:34,567 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,567 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,569 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 13207a781081472a057b14b506529437 2024-11-20T17:50:34,570 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13207a781081472a057b14b506529437 columnFamilyName A 2024-11-20T17:50:34,571 DEBUG [StoreOpener-13207a781081472a057b14b506529437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:34,571 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(327): Store=13207a781081472a057b14b506529437/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:34,571 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 13207a781081472a057b14b506529437 2024-11-20T17:50:34,572 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13207a781081472a057b14b506529437 columnFamilyName B 2024-11-20T17:50:34,572 DEBUG [StoreOpener-13207a781081472a057b14b506529437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:34,572 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(327): Store=13207a781081472a057b14b506529437/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:34,572 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 13207a781081472a057b14b506529437 2024-11-20T17:50:34,573 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13207a781081472a057b14b506529437 columnFamilyName C 2024-11-20T17:50:34,573 DEBUG [StoreOpener-13207a781081472a057b14b506529437-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:34,573 INFO [StoreOpener-13207a781081472a057b14b506529437-1 {}] regionserver.HStore(327): Store=13207a781081472a057b14b506529437/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:50:34,573 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,574 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437 2024-11-20T17:50:34,574 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437 2024-11-20T17:50:34,575 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,575 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,576 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T17:50:34,577 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,577 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened 13207a781081472a057b14b506529437; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72596079, jitterRate=0.08176587522029877}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T17:50:34,578 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13207a781081472a057b14b506529437 2024-11-20T17:50:34,578 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for 13207a781081472a057b14b506529437: Running coprocessor pre-open hook at 1732125034568Writing region info on filesystem at 1732125034568Initializing all the Stores at 1732125034568Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125034568Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125034569 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125034569Cleaning up temporary data from old regions at 1732125034575 (+6 ms)Running coprocessor post-open hooks at 1732125034578 (+3 ms)Region opened successfully at 1732125034578 2024-11-20T17:50:34,579 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., pid=155, masterSystemTime=1732125034564 2024-11-20T17:50:34,580 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,580 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,581 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=OPEN, openSeqNum=5, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:50:34,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-20T17:50:34,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 in 171 msec 2024-11-20T17:50:34,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T17:50:34,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, REOPEN/MOVE in 492 msec 2024-11-20T17:50:34,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-20T17:50:34,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-11-20T17:50:34,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-11-20T17:50:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-20T17:50:34,592 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:34,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2a1e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,593 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52dcc6c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,595 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ae1998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdeed38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,596 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e596e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,598 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f492b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6eabbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,600 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481c2a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520292bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3be2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,604 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef4bc3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,605 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d04293e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,607 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62355a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,609 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b87080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108b0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,611 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee693af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7dc8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5be7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745b9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,618 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac5a611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162f1ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,622 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c889785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,624 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f674a50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e62e123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,626 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3528b26f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6882d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b72907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,39775,-1 for getting cluster id 2024-11-20T17:50:34,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T17:50:34,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1af8210c-6bb5-4317-b742-986938d243d3' 2024-11-20T17:50:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T17:50:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1af8210c-6bb5-4317-b742-986938d243d3" 2024-11-20T17:50:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3746071d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,39775,-1] 2024-11-20T17:50:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T17:50:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:34,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T17:50:34,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7edad271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T17:50:34,633 DEBUG [Thread-2639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,634 DEBUG [Thread-2630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,634 DEBUG [Thread-2624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,634 DEBUG [Thread-2641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,634 DEBUG [Thread-2622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,635 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:34,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,636 DEBUG [Thread-2641 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-20T17:50:34,638 DEBUG [Thread-2641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,638 DEBUG [Thread-2626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:34,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:34,639 DEBUG [Thread-2637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,642 DEBUG [Thread-2628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,644 DEBUG [Thread-2633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,644 DEBUG [Thread-2635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T17:50:34,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,44957,1732124872893, seqNum=-1] 2024-11-20T17:50:34,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T17:50:34,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T17:50:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5] 2024-11-20T17:50:34,660 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:34,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:34,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:34,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:50:34,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200d894785397e481188f7b314e87d749b_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125034657/Put/seqid=0 2024-11-20T17:50:34,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125094695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125094695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125094696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125094699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125094701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742416_1592 (size=12154) 2024-11-20T17:50:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:34,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125094807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125094808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125094808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125094808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125094808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:34,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:34,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:34,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:34,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:34,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:34,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125095017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125095017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125095017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125095017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125095018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,118 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:35,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200d894785397e481188f7b314e87d749b_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200d894785397e481188f7b314e87d749b_13207a781081472a057b14b506529437 2024-11-20T17:50:35,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/cf1c10f8b1b149f7b265ee02935db368, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:35,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:35,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/cf1c10f8b1b149f7b265ee02935db368 is 175, key is test_row_0/A:col10/1732125034657/Put/seqid=0 2024-11-20T17:50:35,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:35,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742417_1593 (size=30955) 2024-11-20T17:50:35,130 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/cf1c10f8b1b149f7b265ee02935db368 2024-11-20T17:50:35,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1baf74d9fe7d4defa6ba5ad803938f83 is 50, key is test_row_0/B:col10/1732125034657/Put/seqid=0 2024-11-20T17:50:35,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742418_1594 (size=12001) 2024-11-20T17:50:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:35,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125095327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125095327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125095327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125095328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125095328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1baf74d9fe7d4defa6ba5ad803938f83 2024-11-20T17:50:35,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/cc11f1c4b3754eefb594c5f6cacd5d69 is 50, key is test_row_0/C:col10/1732125034657/Put/seqid=0 2024-11-20T17:50:35,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742419_1595 (size=12001) 2024-11-20T17:50:35,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:35,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125095837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125095837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125095837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125095837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125095837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:35,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:35,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:35,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:35,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/cc11f1c4b3754eefb594c5f6cacd5d69 2024-11-20T17:50:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/cf1c10f8b1b149f7b265ee02935db368 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368 2024-11-20T17:50:36,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368, entries=150, sequenceid=16, filesize=30.2 K 2024-11-20T17:50:36,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1baf74d9fe7d4defa6ba5ad803938f83 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1baf74d9fe7d4defa6ba5ad803938f83 2024-11-20T17:50:36,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1baf74d9fe7d4defa6ba5ad803938f83, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T17:50:36,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/cc11f1c4b3754eefb594c5f6cacd5d69 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/cc11f1c4b3754eefb594c5f6cacd5d69 2024-11-20T17:50:36,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/cc11f1c4b3754eefb594c5f6cacd5d69, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T17:50:36,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 13207a781081472a057b14b506529437 in 1353ms, sequenceid=16, compaction requested=false 2024-11-20T17:50:36,015 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T17:50:36,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:36,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T17:50:36,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:36,044 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:50:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112061582b8dff074642ba0db947957f58f0_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125034698/Put/seqid=0 2024-11-20T17:50:36,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742420_1596 (size=12154) 2024-11-20T17:50:36,261 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T17:50:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:36,463 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112061582b8dff074642ba0db947957f58f0_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112061582b8dff074642ba0db947957f58f0_13207a781081472a057b14b506529437 2024-11-20T17:50:36,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/1fba23047b7a4f13b28958fb2a8d62b0, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:36,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/1fba23047b7a4f13b28958fb2a8d62b0 is 175, key is test_row_0/A:col10/1732125034698/Put/seqid=0 2024-11-20T17:50:36,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742421_1597 (size=30955) 2024-11-20T17:50:36,471 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/1fba23047b7a4f13b28958fb2a8d62b0 2024-11-20T17:50:36,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/bb4b8c0105fc475ab2dca6627b18b570 is 50, key is test_row_0/B:col10/1732125034698/Put/seqid=0 2024-11-20T17:50:36,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742422_1598 (size=12001) 2024-11-20T17:50:36,491 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/bb4b8c0105fc475ab2dca6627b18b570 2024-11-20T17:50:36,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/7a6079d303c94068970ea5c48b99eddd is 50, key is test_row_0/C:col10/1732125034698/Put/seqid=0 2024-11-20T17:50:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742423_1599 (size=12001) 2024-11-20T17:50:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:36,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125096857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:36,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125096857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:36,861 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:36,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125096858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:36,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:36,863 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125096859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125096860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:36,916 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/7a6079d303c94068970ea5c48b99eddd 2024-11-20T17:50:36,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/1fba23047b7a4f13b28958fb2a8d62b0 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0 2024-11-20T17:50:36,935 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0, entries=150, sequenceid=40, filesize=30.2 K 2024-11-20T17:50:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/bb4b8c0105fc475ab2dca6627b18b570 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/bb4b8c0105fc475ab2dca6627b18b570 2024-11-20T17:50:36,940 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/bb4b8c0105fc475ab2dca6627b18b570, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T17:50:36,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/7a6079d303c94068970ea5c48b99eddd as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/7a6079d303c94068970ea5c48b99eddd 2024-11-20T17:50:36,947 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/7a6079d303c94068970ea5c48b99eddd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T17:50:36,949 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 13207a781081472a057b14b506529437 in 905ms, sequenceid=40, compaction requested=false 2024-11-20T17:50:36,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:36,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:36,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-20T17:50:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-20T17:50:36,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T17:50:36,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2920 sec 2024-11-20T17:50:36,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.3230 sec 2024-11-20T17:50:36,970 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:36,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:50:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:36,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203239120fc3a04df3aabd0c0f673bac63_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:36,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742424_1600 (size=14594) 2024-11-20T17:50:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125097011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125097012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125097015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125097117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125097118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125097127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125097327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125097328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125097337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:37,398 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203239120fc3a04df3aabd0c0f673bac63_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203239120fc3a04df3aabd0c0f673bac63_13207a781081472a057b14b506529437 2024-11-20T17:50:37,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f5e676d0a777419c8adb3ce1cac322f9, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:37,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f5e676d0a777419c8adb3ce1cac322f9 is 175, key is test_row_0/A:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:37,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742425_1601 (size=39549) 2024-11-20T17:50:37,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f5e676d0a777419c8adb3ce1cac322f9 2024-11-20T17:50:37,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/461975d1563a466e8f4b10dac60ce586 is 50, key is test_row_0/B:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:37,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742426_1602 (size=12001) 2024-11-20T17:50:37,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/461975d1563a466e8f4b10dac60ce586 2024-11-20T17:50:37,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/5613a279ae4241a9b0c9ce4e837fa264 is 50, key is test_row_0/C:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:37,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742427_1603 (size=12001) 2024-11-20T17:50:37,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/5613a279ae4241a9b0c9ce4e837fa264 2024-11-20T17:50:37,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f5e676d0a777419c8adb3ce1cac322f9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9 2024-11-20T17:50:37,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9, entries=200, sequenceid=55, filesize=38.6 K 2024-11-20T17:50:37,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/461975d1563a466e8f4b10dac60ce586 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/461975d1563a466e8f4b10dac60ce586 2024-11-20T17:50:37,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/461975d1563a466e8f4b10dac60ce586, entries=150, sequenceid=55, filesize=11.7 K 2024-11-20T17:50:37,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/5613a279ae4241a9b0c9ce4e837fa264 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5613a279ae4241a9b0c9ce4e837fa264 2024-11-20T17:50:37,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5613a279ae4241a9b0c9ce4e837fa264, entries=150, sequenceid=55, filesize=11.7 K 2024-11-20T17:50:37,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 13207a781081472a057b14b506529437 in 493ms, sequenceid=55, compaction requested=true 2024-11-20T17:50:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:37,463 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:37,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:37,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:37,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T17:50:37,464 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:37,464 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:37,464 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:37,465 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:37,465 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:37,465 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:37,465 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=99.1 K 2024-11-20T17:50:37,465 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:37,465 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:37,465 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1baf74d9fe7d4defa6ba5ad803938f83, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/bb4b8c0105fc475ab2dca6627b18b570, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/461975d1563a466e8f4b10dac60ce586] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=35.2 K 2024-11-20T17:50:37,465 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9] 2024-11-20T17:50:37,465 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1baf74d9fe7d4defa6ba5ad803938f83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732125034657 2024-11-20T17:50:37,465 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf1c10f8b1b149f7b265ee02935db368, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732125034657 2024-11-20T17:50:37,465 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting bb4b8c0105fc475ab2dca6627b18b570, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732125034695 2024-11-20T17:50:37,466 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fba23047b7a4f13b28958fb2a8d62b0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732125034695 2024-11-20T17:50:37,466 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 461975d1563a466e8f4b10dac60ce586, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732125036854 2024-11-20T17:50:37,466 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e676d0a777419c8adb3ce1cac322f9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732125036854 2024-11-20T17:50:37,474 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:37,475 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:37,476 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/3c3019e1098747eb981e0d9e85675b8f is 50, key is test_row_0/B:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:37,480 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411204b7220e3ac1348328d93c601de9c592c_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:37,482 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411204b7220e3ac1348328d93c601de9c592c_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:37,483 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204b7220e3ac1348328d93c601de9c592c_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:37,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742428_1604 (size=12104) 2024-11-20T17:50:37,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742429_1605 (size=4469) 2024-11-20T17:50:37,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:37,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:37,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T17:50:37,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205349ccce668a4df190f7999d6809b1f8_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125037009/Put/seqid=0 2024-11-20T17:50:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742430_1606 (size=12154) 2024-11-20T17:50:37,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125097705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125097707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125097707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125097817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125097817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125097818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:37,892 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:37,893 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/dd672781248240408a1eb629a91bda89 is 175, key is test_row_0/A:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:37,894 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/3c3019e1098747eb981e0d9e85675b8f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3c3019e1098747eb981e0d9e85675b8f 2024-11-20T17:50:37,898 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into 3c3019e1098747eb981e0d9e85675b8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:37,898 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:37,898 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=13, startTime=1732125037463; duration=0sec 2024-11-20T17:50:37,898 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:37,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:37,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:37,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:37,899 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:37,900 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:37,900 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/cc11f1c4b3754eefb594c5f6cacd5d69, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/7a6079d303c94068970ea5c48b99eddd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5613a279ae4241a9b0c9ce4e837fa264] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=35.2 K 2024-11-20T17:50:37,900 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting cc11f1c4b3754eefb594c5f6cacd5d69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732125034657 2024-11-20T17:50:37,900 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a6079d303c94068970ea5c48b99eddd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732125034695 2024-11-20T17:50:37,900 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5613a279ae4241a9b0c9ce4e837fa264, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732125036854 2024-11-20T17:50:37,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742431_1607 (size=31058) 2024-11-20T17:50:37,907 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/dd672781248240408a1eb629a91bda89 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/dd672781248240408a1eb629a91bda89 2024-11-20T17:50:37,909 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:37,910 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/c0cd668aa3084cb79f26a968f3fc2299 is 50, key is test_row_0/C:col10/1732125036968/Put/seqid=0 2024-11-20T17:50:37,912 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into dd672781248240408a1eb629a91bda89(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:37,912 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:37,912 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=13, startTime=1732125037463; duration=0sec 2024-11-20T17:50:37,913 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:37,913 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:50:37,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742432_1608 (size=12104) 2024-11-20T17:50:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125098027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125098028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125098028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:38,069 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205349ccce668a4df190f7999d6809b1f8_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205349ccce668a4df190f7999d6809b1f8_13207a781081472a057b14b506529437 2024-11-20T17:50:38,070 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/66e383209d674045b4b817290baf3e37, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:38,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/66e383209d674045b4b817290baf3e37 is 175, key is test_row_0/A:col10/1732125037009/Put/seqid=0 2024-11-20T17:50:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742433_1609 (size=30955) 2024-11-20T17:50:38,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/66e383209d674045b4b817290baf3e37 2024-11-20T17:50:38,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/875c61102a764481b40222061f191547 is 50, key is test_row_0/B:col10/1732125037009/Put/seqid=0 2024-11-20T17:50:38,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742434_1610 (size=12001) 2024-11-20T17:50:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125098337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125098337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125098337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,352 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/c0cd668aa3084cb79f26a968f3fc2299 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/c0cd668aa3084cb79f26a968f3fc2299 2024-11-20T17:50:38,356 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into c0cd668aa3084cb79f26a968f3fc2299(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:38,356 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:38,356 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=13, startTime=1732125037464; duration=0sec 2024-11-20T17:50:38,356 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:38,356 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:38,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/875c61102a764481b40222061f191547 2024-11-20T17:50:38,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/98b6aa5132ec48628ebf321dc7aaf69f is 50, key is test_row_0/C:col10/1732125037009/Put/seqid=0 2024-11-20T17:50:38,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742435_1611 (size=12001) 2024-11-20T17:50:38,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/98b6aa5132ec48628ebf321dc7aaf69f 2024-11-20T17:50:38,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/66e383209d674045b4b817290baf3e37 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37 2024-11-20T17:50:38,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37, entries=150, sequenceid=77, filesize=30.2 K 2024-11-20T17:50:38,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/875c61102a764481b40222061f191547 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/875c61102a764481b40222061f191547 2024-11-20T17:50:38,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/875c61102a764481b40222061f191547, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T17:50:38,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/98b6aa5132ec48628ebf321dc7aaf69f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/98b6aa5132ec48628ebf321dc7aaf69f 2024-11-20T17:50:38,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/98b6aa5132ec48628ebf321dc7aaf69f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T17:50:38,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 13207a781081472a057b14b506529437 in 902ms, sequenceid=77, compaction requested=false 2024-11-20T17:50:38,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T17:50:38,797 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:38,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-20T17:50:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:38,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:38,800 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:38,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:38,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:38,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T17:50:38,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f1ded8539559495d81681fa7f61250d4_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742436_1612 (size=12154) 2024-11-20T17:50:38,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:38,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f1ded8539559495d81681fa7f61250d4_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f1ded8539559495d81681fa7f61250d4_13207a781081472a057b14b506529437 2024-11-20T17:50:38,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ebdc330d877a4976b7959e2b70a429e2, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:38,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ebdc330d877a4976b7959e2b70a429e2 is 175, key is test_row_0/A:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:38,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742437_1613 (size=30955) 2024-11-20T17:50:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125098882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125098882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125098883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125098884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125098884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:38,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:38,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:38,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:38,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:38,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:38,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:38,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125098997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125098997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125098997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125098997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125098997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:39,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:39,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:39,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125099207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125099208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125099208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125099208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125099208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:39,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:39,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ebdc330d877a4976b7959e2b70a429e2 2024-11-20T17:50:39,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d915217777f34c31b113b6d6d1af9dbf is 50, key is test_row_0/B:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:39,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742438_1614 (size=12001) 2024-11-20T17:50:39,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d915217777f34c31b113b6d6d1af9dbf 2024-11-20T17:50:39,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b84889fc0832430587d30bf4898de258 is 50, key is test_row_0/C:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:39,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742439_1615 (size=12001) 2024-11-20T17:50:39,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:39,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:39,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125099517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125099517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125099517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125099517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125099517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:39,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:39,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:39,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b84889fc0832430587d30bf4898de258 2024-11-20T17:50:39,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ebdc330d877a4976b7959e2b70a429e2 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2 2024-11-20T17:50:39,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2, entries=150, sequenceid=97, filesize=30.2 K 2024-11-20T17:50:39,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d915217777f34c31b113b6d6d1af9dbf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d915217777f34c31b113b6d6d1af9dbf 2024-11-20T17:50:39,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:39,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d915217777f34c31b113b6d6d1af9dbf, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T17:50:39,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b84889fc0832430587d30bf4898de258 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b84889fc0832430587d30bf4898de258 2024-11-20T17:50:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:39,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b84889fc0832430587d30bf4898de258, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T17:50:39,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 13207a781081472a057b14b506529437 in 887ms, sequenceid=97, compaction requested=true 2024-11-20T17:50:39,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:39,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:39,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:39,735 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:39,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:39,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:39,736 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:39,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:39,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:39,737 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:39,737 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:39,737 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:39,737 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:39,737 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,737 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,737 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/dd672781248240408a1eb629a91bda89, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=90.8 K 2024-11-20T17:50:39,737 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3c3019e1098747eb981e0d9e85675b8f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/875c61102a764481b40222061f191547, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d915217777f34c31b113b6d6d1af9dbf] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=35.3 K 2024-11-20T17:50:39,737 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,737 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/dd672781248240408a1eb629a91bda89, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2] 2024-11-20T17:50:39,738 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd672781248240408a1eb629a91bda89, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732125036854 2024-11-20T17:50:39,738 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3019e1098747eb981e0d9e85675b8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732125036854 2024-11-20T17:50:39,738 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 875c61102a764481b40222061f191547, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732125037009 2024-11-20T17:50:39,738 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66e383209d674045b4b817290baf3e37, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732125037009 2024-11-20T17:50:39,738 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d915217777f34c31b113b6d6d1af9dbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732125038847 2024-11-20T17:50:39,738 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebdc330d877a4976b7959e2b70a429e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732125038847 2024-11-20T17:50:39,753 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:39,763 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:39,764 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/96b50204f9504335a5276091b3327fcc is 50, key is test_row_0/B:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:39,765 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411207adfa9d7dffa4140a62577f447c23239_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:39,767 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411207adfa9d7dffa4140a62577f447c23239_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:39,767 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207adfa9d7dffa4140a62577f447c23239_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:39,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742441_1617 (size=4469) 2024-11-20T17:50:39,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742440_1616 (size=12207) 2024-11-20T17:50:39,779 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#521 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:39,780 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/faeba2a4e0494e31bead1e7f2176fb25 is 175, key is test_row_0/A:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:39,782 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/96b50204f9504335a5276091b3327fcc as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/96b50204f9504335a5276091b3327fcc 2024-11-20T17:50:39,786 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into 96b50204f9504335a5276091b3327fcc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:39,786 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:39,786 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=13, startTime=1732125039735; duration=0sec 2024-11-20T17:50:39,786 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:39,786 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:39,786 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:39,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:39,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:39,788 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,788 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/c0cd668aa3084cb79f26a968f3fc2299, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/98b6aa5132ec48628ebf321dc7aaf69f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b84889fc0832430587d30bf4898de258] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=35.3 K 2024-11-20T17:50:39,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting c0cd668aa3084cb79f26a968f3fc2299, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732125036854 2024-11-20T17:50:39,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b6aa5132ec48628ebf321dc7aaf69f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732125037009 2024-11-20T17:50:39,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b84889fc0832430587d30bf4898de258, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732125038847 2024-11-20T17:50:39,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742442_1618 (size=31161) 2024-11-20T17:50:39,800 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/faeba2a4e0494e31bead1e7f2176fb25 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/faeba2a4e0494e31bead1e7f2176fb25 2024-11-20T17:50:39,803 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:39,803 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/8a9b3bb611f8496fa334664ae61ac232 is 50, key is test_row_0/C:col10/1732125038848/Put/seqid=0 2024-11-20T17:50:39,806 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into faeba2a4e0494e31bead1e7f2176fb25(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:39,806 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:39,806 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=13, startTime=1732125039735; duration=0sec 2024-11-20T17:50:39,806 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:39,806 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:50:39,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742443_1619 (size=12207) 2024-11-20T17:50:39,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T17:50:39,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:39,874 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T17:50:39,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120264955628baf40339a24c002e55535ee_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125038879/Put/seqid=0 2024-11-20T17:50:39,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742444_1620 (size=12154) 2024-11-20T17:50:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:40,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:40,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125100049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125100049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125100050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125100050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125100050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125100157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125100157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125100157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125100167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125100167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,227 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/8a9b3bb611f8496fa334664ae61ac232 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/8a9b3bb611f8496fa334664ae61ac232 2024-11-20T17:50:40,230 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into 8a9b3bb611f8496fa334664ae61ac232(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:40,231 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:40,231 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=13, startTime=1732125039735; duration=0sec 2024-11-20T17:50:40,231 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:40,231 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:40,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:40,298 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120264955628baf40339a24c002e55535ee_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120264955628baf40339a24c002e55535ee_13207a781081472a057b14b506529437 2024-11-20T17:50:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b4a56b134a2a426096ea4c8f7436b437, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:40,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b4a56b134a2a426096ea4c8f7436b437 is 175, key is test_row_0/A:col10/1732125038879/Put/seqid=0 2024-11-20T17:50:40,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742445_1621 (size=30955) 2024-11-20T17:50:40,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125100367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125100367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125100367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125100377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125100378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125100677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125100677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125100677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125100687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125100687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:40,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:41,062 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b4a56b134a2a426096ea4c8f7436b437 2024-11-20T17:50:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/e293943852564f38a9b07653f6e7fa55 is 50, key is test_row_0/B:col10/1732125038879/Put/seqid=0 2024-11-20T17:50:41,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742446_1622 (size=12001) 2024-11-20T17:50:41,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125101187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125101188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:41,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125101188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125101197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125101197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:41,474 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/e293943852564f38a9b07653f6e7fa55 2024-11-20T17:50:41,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/31de302da2b143e79fae5fb633ec373c is 50, key is test_row_0/C:col10/1732125038879/Put/seqid=0 2024-11-20T17:50:41,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742447_1623 (size=12001) 2024-11-20T17:50:41,888 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/31de302da2b143e79fae5fb633ec373c 2024-11-20T17:50:41,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b4a56b134a2a426096ea4c8f7436b437 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437 2024-11-20T17:50:41,897 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437, entries=150, sequenceid=119, filesize=30.2 K 2024-11-20T17:50:41,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/e293943852564f38a9b07653f6e7fa55 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/e293943852564f38a9b07653f6e7fa55 2024-11-20T17:50:41,901 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/e293943852564f38a9b07653f6e7fa55, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T17:50:41,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/31de302da2b143e79fae5fb633ec373c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/31de302da2b143e79fae5fb633ec373c 2024-11-20T17:50:41,904 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/31de302da2b143e79fae5fb633ec373c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T17:50:41,905 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 13207a781081472a057b14b506529437 in 2031ms, sequenceid=119, compaction requested=false 2024-11-20T17:50:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-20T17:50:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-20T17:50:41,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-20T17:50:41,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1060 sec 2024-11-20T17:50:41,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.1100 sec 2024-11-20T17:50:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:42,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:42,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T17:50:42,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d76b096d959840a38a34d275c46f5d70_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742448_1624 (size=12304) 2024-11-20T17:50:42,214 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:42,218 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d76b096d959840a38a34d275c46f5d70_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d76b096d959840a38a34d275c46f5d70_13207a781081472a057b14b506529437 2024-11-20T17:50:42,220 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/08b9e8cbf9ac4a88896059e48aeb57f6, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:42,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/08b9e8cbf9ac4a88896059e48aeb57f6 is 175, key is test_row_0/A:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125102218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125102220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125102221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125102222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125102223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742449_1625 (size=31105) 2024-11-20T17:50:42,230 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/08b9e8cbf9ac4a88896059e48aeb57f6 2024-11-20T17:50:42,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/48ebbe7ff3244760bb340c39f693e5ae is 50, key is test_row_0/B:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:42,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742450_1626 (size=12151) 2024-11-20T17:50:42,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/48ebbe7ff3244760bb340c39f693e5ae 2024-11-20T17:50:42,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/41b0f8cab76c4c55b67f147a465c7705 is 50, key is test_row_0/C:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:42,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742451_1627 (size=12151) 2024-11-20T17:50:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125102328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125102328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125102328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125102337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125102337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,505 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T17:50:42,505 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T17:50:42,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125102541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125102541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125102541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125102547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125102547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/41b0f8cab76c4c55b67f147a465c7705 2024-11-20T17:50:42,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/08b9e8cbf9ac4a88896059e48aeb57f6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6 2024-11-20T17:50:42,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6, entries=150, sequenceid=140, filesize=30.4 K 2024-11-20T17:50:42,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/48ebbe7ff3244760bb340c39f693e5ae as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/48ebbe7ff3244760bb340c39f693e5ae 2024-11-20T17:50:42,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/48ebbe7ff3244760bb340c39f693e5ae, entries=150, sequenceid=140, filesize=11.9 K 2024-11-20T17:50:42,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/41b0f8cab76c4c55b67f147a465c7705 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/41b0f8cab76c4c55b67f147a465c7705 2024-11-20T17:50:42,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/41b0f8cab76c4c55b67f147a465c7705, entries=150, sequenceid=140, filesize=11.9 K 2024-11-20T17:50:42,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 13207a781081472a057b14b506529437 in 491ms, sequenceid=140, compaction requested=true 2024-11-20T17:50:42,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:42,691 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:42,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:42,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:42,692 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:42,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:42,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:42,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:42,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:42,693 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:42,693 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/faeba2a4e0494e31bead1e7f2176fb25, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=91.0 K 2024-11-20T17:50:42,693 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/faeba2a4e0494e31bead1e7f2176fb25, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6] 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting faeba2a4e0494e31bead1e7f2176fb25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732125038847 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:42,693 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:42,693 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/96b50204f9504335a5276091b3327fcc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/e293943852564f38a9b07653f6e7fa55, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/48ebbe7ff3244760bb340c39f693e5ae] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=35.5 K 2024-11-20T17:50:42,693 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a56b134a2a426096ea4c8f7436b437, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732125038879 2024-11-20T17:50:42,694 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b50204f9504335a5276091b3327fcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732125038847 2024-11-20T17:50:42,694 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b9e8cbf9ac4a88896059e48aeb57f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732125040046 2024-11-20T17:50:42,694 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting e293943852564f38a9b07653f6e7fa55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732125038879 2024-11-20T17:50:42,695 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ebbe7ff3244760bb340c39f693e5ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732125040046 2024-11-20T17:50:42,703 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:42,704 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/3f4a0d6e33024d699d564ccfdfc23233 is 50, key is test_row_0/B:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:42,713 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:42,725 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120fb2aaa3ed8bb46be8bc3a2ba0411d638_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:42,727 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120fb2aaa3ed8bb46be8bc3a2ba0411d638_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:42,727 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fb2aaa3ed8bb46be8bc3a2ba0411d638_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:42,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742452_1628 (size=12459) 2024-11-20T17:50:42,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742453_1629 (size=4469) 2024-11-20T17:50:42,753 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/3f4a0d6e33024d699d564ccfdfc23233 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3f4a0d6e33024d699d564ccfdfc23233 2024-11-20T17:50:42,759 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into 3f4a0d6e33024d699d564ccfdfc23233(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:42,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:42,759 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=13, startTime=1732125042692; duration=0sec 2024-11-20T17:50:42,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:42,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:42,759 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:42,761 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:42,761 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:42,761 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:42,762 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/8a9b3bb611f8496fa334664ae61ac232, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/31de302da2b143e79fae5fb633ec373c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/41b0f8cab76c4c55b67f147a465c7705] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=35.5 K 2024-11-20T17:50:42,762 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9b3bb611f8496fa334664ae61ac232, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732125038847 2024-11-20T17:50:42,762 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 31de302da2b143e79fae5fb633ec373c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732125038879 2024-11-20T17:50:42,763 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b0f8cab76c4c55b67f147a465c7705, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732125040046 2024-11-20T17:50:42,778 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:42,779 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/649a5dab3c2247fca5480bba7cf1a71d is 50, key is test_row_0/C:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:42,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742454_1630 (size=12459) 2024-11-20T17:50:42,788 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/649a5dab3c2247fca5480bba7cf1a71d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/649a5dab3c2247fca5480bba7cf1a71d 2024-11-20T17:50:42,794 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into 649a5dab3c2247fca5480bba7cf1a71d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:42,794 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:42,794 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=13, startTime=1732125042692; duration=0sec 2024-11-20T17:50:42,794 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:42,794 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:42,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:42,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T17:50:42,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120acf7c428ba5e457988a59a9c4b5adc20_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125042215/Put/seqid=0 2024-11-20T17:50:42,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742455_1631 (size=14794) 2024-11-20T17:50:42,863 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:42,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120acf7c428ba5e457988a59a9c4b5adc20_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120acf7c428ba5e457988a59a9c4b5adc20_13207a781081472a057b14b506529437 2024-11-20T17:50:42,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/864b48ae3ffa4819a44cb8345006d75c, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:42,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/864b48ae3ffa4819a44cb8345006d75c is 175, key is test_row_0/A:col10/1732125042215/Put/seqid=0 2024-11-20T17:50:42,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742456_1632 (size=39749) 2024-11-20T17:50:42,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/864b48ae3ffa4819a44cb8345006d75c 2024-11-20T17:50:42,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125102872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/9bebc37af26f4945b96c819712fdfd61 is 50, key is test_row_0/B:col10/1732125042215/Put/seqid=0 2024-11-20T17:50:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125102874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125102875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125102875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125102877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742457_1633 (size=12151) 2024-11-20T17:50:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T17:50:42,968 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:42,968 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-20T17:50:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T17:50:42,970 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:42,971 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:42,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125102987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125102987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125102987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125102987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125102987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:42,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T17:50:43,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T17:50:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:43,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:43,150 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#531 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:43,150 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/15f8c571b4784d9aa7def42d02fa1660 is 175, key is test_row_0/A:col10/1732125042199/Put/seqid=0 2024-11-20T17:50:43,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742458_1634 (size=31413) 2024-11-20T17:50:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125103197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125103197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125103197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125103198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125103198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T17:50:43,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:43,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:43,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:43,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:43,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T17:50:43,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/9bebc37af26f4945b96c819712fdfd61 2024-11-20T17:50:43,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b1882bc024284f59ba2063f3f6012b35 is 50, key is test_row_0/C:col10/1732125042215/Put/seqid=0 2024-11-20T17:50:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742459_1635 (size=12151) 2024-11-20T17:50:43,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b1882bc024284f59ba2063f3f6012b35 2024-11-20T17:50:43,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/864b48ae3ffa4819a44cb8345006d75c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c 2024-11-20T17:50:43,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c, entries=200, sequenceid=159, filesize=38.8 K 2024-11-20T17:50:43,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/9bebc37af26f4945b96c819712fdfd61 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/9bebc37af26f4945b96c819712fdfd61 2024-11-20T17:50:43,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/9bebc37af26f4945b96c819712fdfd61, entries=150, sequenceid=159, filesize=11.9 K 2024-11-20T17:50:43,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b1882bc024284f59ba2063f3f6012b35 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b1882bc024284f59ba2063f3f6012b35 2024-11-20T17:50:43,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b1882bc024284f59ba2063f3f6012b35, entries=150, sequenceid=159, filesize=11.9 K 2024-11-20T17:50:43,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 13207a781081472a057b14b506529437 in 474ms, sequenceid=159, compaction requested=false 2024-11-20T17:50:43,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:43,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T17:50:43,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:43,428 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T17:50:43,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120076ecac078504f9894f44dcf20e03e20_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125042868/Put/seqid=0 2024-11-20T17:50:43,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742460_1636 (size=12304) 2024-11-20T17:50:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:43,446 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120076ecac078504f9894f44dcf20e03e20_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120076ecac078504f9894f44dcf20e03e20_13207a781081472a057b14b506529437 2024-11-20T17:50:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6c003b98d0684769b250dcd72364a4bb, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6c003b98d0684769b250dcd72364a4bb is 175, key is test_row_0/A:col10/1732125042868/Put/seqid=0 2024-11-20T17:50:43,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742461_1637 (size=31105) 2024-11-20T17:50:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125103522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125103523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125103525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125103527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125103529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,563 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/15f8c571b4784d9aa7def42d02fa1660 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/15f8c571b4784d9aa7def42d02fa1660 2024-11-20T17:50:43,569 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into 15f8c571b4784d9aa7def42d02fa1660(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:43,569 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:43,569 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=13, startTime=1732125042691; duration=0sec 2024-11-20T17:50:43,569 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:43,569 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:50:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T17:50:43,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125103637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125103637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125103638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125103638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125103638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125103847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125103847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125103848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125103848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125103848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:43,858 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6c003b98d0684769b250dcd72364a4bb 2024-11-20T17:50:43,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d7648183ebf64ca8a7394aad92c50b9c is 50, key is test_row_0/B:col10/1732125042868/Put/seqid=0 2024-11-20T17:50:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742462_1638 (size=12151) 2024-11-20T17:50:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T17:50:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125104157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125104157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125104157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125104157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125104157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,282 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d7648183ebf64ca8a7394aad92c50b9c 2024-11-20T17:50:44,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/ff9a8e823d0d4dcf9bdf53e46d089d86 is 50, key is test_row_0/C:col10/1732125042868/Put/seqid=0 2024-11-20T17:50:44,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742463_1639 (size=12151) 2024-11-20T17:50:44,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125104667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125104667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125104667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125104667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125104677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:44,698 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/ff9a8e823d0d4dcf9bdf53e46d089d86 2024-11-20T17:50:44,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6c003b98d0684769b250dcd72364a4bb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb 2024-11-20T17:50:44,712 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb, entries=150, sequenceid=178, filesize=30.4 K 2024-11-20T17:50:44,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d7648183ebf64ca8a7394aad92c50b9c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d7648183ebf64ca8a7394aad92c50b9c 2024-11-20T17:50:44,718 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d7648183ebf64ca8a7394aad92c50b9c, entries=150, sequenceid=178, filesize=11.9 K 2024-11-20T17:50:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/ff9a8e823d0d4dcf9bdf53e46d089d86 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/ff9a8e823d0d4dcf9bdf53e46d089d86 2024-11-20T17:50:44,722 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/ff9a8e823d0d4dcf9bdf53e46d089d86, entries=150, sequenceid=178, filesize=11.9 K 2024-11-20T17:50:44,723 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 13207a781081472a057b14b506529437 in 1295ms, sequenceid=178, compaction requested=true 2024-11-20T17:50:44,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:44,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:44,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-20T17:50:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-20T17:50:44,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T17:50:44,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7540 sec 2024-11-20T17:50:44,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7590 sec 2024-11-20T17:50:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T17:50:45,118 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:45,118 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-20T17:50:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T17:50:45,120 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:45,121 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:45,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T17:50:45,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T17:50:45,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:45,278 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T17:50:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112070e66cbd9975448385d47bdfbb6de9a0_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125043519/Put/seqid=0 2024-11-20T17:50:45,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742464_1640 (size=12304) 2024-11-20T17:50:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T17:50:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:45,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125105697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,703 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125105700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125105701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125105703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125105705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:45,745 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112070e66cbd9975448385d47bdfbb6de9a0_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112070e66cbd9975448385d47bdfbb6de9a0_13207a781081472a057b14b506529437 2024-11-20T17:50:45,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ac5ff601dbc94148b158d7b910acd6c9, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:45,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ac5ff601dbc94148b158d7b910acd6c9 is 175, key is test_row_0/A:col10/1732125043519/Put/seqid=0 2024-11-20T17:50:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T17:50:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742465_1641 (size=31105) 2024-11-20T17:50:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125105817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125105817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125105817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125105817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125106027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125106027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125106027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125106027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,170 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ac5ff601dbc94148b158d7b910acd6c9 2024-11-20T17:50:46,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/743f681703a34f38ae2de6880a78229c is 50, key is test_row_0/B:col10/1732125043519/Put/seqid=0 2024-11-20T17:50:46,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742466_1642 (size=12151) 2024-11-20T17:50:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T17:50:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125106337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125106337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125106337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125106338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,587 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/743f681703a34f38ae2de6880a78229c 2024-11-20T17:50:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/f8767e92620b42539267d3c5fa14f111 is 50, key is test_row_0/C:col10/1732125043519/Put/seqid=0 2024-11-20T17:50:46,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742467_1643 (size=12151) 2024-11-20T17:50:46,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125106848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125106848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125106848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125106848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:46,996 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/f8767e92620b42539267d3c5fa14f111 2024-11-20T17:50:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/ac5ff601dbc94148b158d7b910acd6c9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9 2024-11-20T17:50:47,004 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9, entries=150, sequenceid=197, filesize=30.4 K 2024-11-20T17:50:47,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/743f681703a34f38ae2de6880a78229c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/743f681703a34f38ae2de6880a78229c 2024-11-20T17:50:47,008 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/743f681703a34f38ae2de6880a78229c, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T17:50:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/f8767e92620b42539267d3c5fa14f111 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f8767e92620b42539267d3c5fa14f111 2024-11-20T17:50:47,012 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f8767e92620b42539267d3c5fa14f111, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T17:50:47,013 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 13207a781081472a057b14b506529437 in 1734ms, sequenceid=197, compaction requested=true 2024-11-20T17:50:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:47,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-20T17:50:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-20T17:50:47,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-20T17:50:47,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8930 sec 2024-11-20T17:50:47,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8980 sec 2024-11-20T17:50:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T17:50:47,268 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:47,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-20T17:50:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T17:50:47,271 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:47,272 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:47,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T17:50:47,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T17:50:47,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:47,425 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T17:50:47,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e9f615866cb47aca8e4e5150ffbd116_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125045701/Put/seqid=0 2024-11-20T17:50:47,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742468_1644 (size=12304) 2024-11-20T17:50:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T17:50:47,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:47,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125107820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:47,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125107857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:47,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125107857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:47,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:47,863 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125107857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:47,863 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:47,864 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:47,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125107867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:47,872 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:47,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:47,890 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e9f615866cb47aca8e4e5150ffbd116_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e9f615866cb47aca8e4e5150ffbd116_13207a781081472a057b14b506529437 2024-11-20T17:50:47,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/a5434d5167ff4e3897283e636b91be65, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:47,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/a5434d5167ff4e3897283e636b91be65 is 175, key is test_row_0/A:col10/1732125045701/Put/seqid=0 2024-11-20T17:50:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T17:50:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742469_1645 (size=31105) 2024-11-20T17:50:47,928 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/a5434d5167ff4e3897283e636b91be65 2024-11-20T17:50:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125107937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:47,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/5e71a118304a4281bebe5ecde59e1d55 is 50, key is test_row_0/B:col10/1732125045701/Put/seqid=0 2024-11-20T17:50:47,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742470_1646 (size=12151) 2024-11-20T17:50:47,957 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/5e71a118304a4281bebe5ecde59e1d55 2024-11-20T17:50:47,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b5ed540c4f194537b6633d61c8a1f081 is 50, key is test_row_0/C:col10/1732125045701/Put/seqid=0 2024-11-20T17:50:48,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742471_1647 (size=12151) 2024-11-20T17:50:48,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125108147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T17:50:48,419 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b5ed540c4f194537b6633d61c8a1f081 2024-11-20T17:50:48,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/a5434d5167ff4e3897283e636b91be65 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65 2024-11-20T17:50:48,433 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65, entries=150, sequenceid=216, filesize=30.4 K 2024-11-20T17:50:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/5e71a118304a4281bebe5ecde59e1d55 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/5e71a118304a4281bebe5ecde59e1d55 2024-11-20T17:50:48,447 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/5e71a118304a4281bebe5ecde59e1d55, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T17:50:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b5ed540c4f194537b6633d61c8a1f081 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b5ed540c4f194537b6633d61c8a1f081 2024-11-20T17:50:48,453 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b5ed540c4f194537b6633d61c8a1f081, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T17:50:48,454 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 13207a781081472a057b14b506529437 in 1028ms, sequenceid=216, compaction requested=true 2024-11-20T17:50:48,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:48,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:48,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-20T17:50:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-20T17:50:48,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-20T17:50:48,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-20T17:50:48,462 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:48,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T17:50:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:48,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.1960 sec 2024-11-20T17:50:48,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120846597cbe20e43febf9ada058ccadb63_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742472_1648 (size=14794) 2024-11-20T17:50:48,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:48,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120846597cbe20e43febf9ada058ccadb63_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120846597cbe20e43febf9ada058ccadb63_13207a781081472a057b14b506529437 2024-11-20T17:50:48,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/bcb58e096fe445a7b4f05f31b8a4c8a4, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:48,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/bcb58e096fe445a7b4f05f31b8a4c8a4 is 175, key is test_row_0/A:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742473_1649 (size=39749) 2024-11-20T17:50:48,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/bcb58e096fe445a7b4f05f31b8a4c8a4 2024-11-20T17:50:48,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d2efbc13ff924947b31dd61f301c5218 is 50, key is test_row_0/B:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742474_1650 (size=12151) 2024-11-20T17:50:48,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d2efbc13ff924947b31dd61f301c5218 2024-11-20T17:50:48,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/f125a5ed58e34d00baa8fb6595afdaa9 is 50, key is test_row_0/C:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125108610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:48,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742475_1651 (size=12151) 2024-11-20T17:50:48,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/f125a5ed58e34d00baa8fb6595afdaa9 2024-11-20T17:50:48,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/bcb58e096fe445a7b4f05f31b8a4c8a4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4 2024-11-20T17:50:48,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4, entries=200, sequenceid=234, filesize=38.8 K 2024-11-20T17:50:48,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d2efbc13ff924947b31dd61f301c5218 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d2efbc13ff924947b31dd61f301c5218 2024-11-20T17:50:48,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d2efbc13ff924947b31dd61f301c5218, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T17:50:48,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/f125a5ed58e34d00baa8fb6595afdaa9 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f125a5ed58e34d00baa8fb6595afdaa9 2024-11-20T17:50:48,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f125a5ed58e34d00baa8fb6595afdaa9, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T17:50:48,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 13207a781081472a057b14b506529437 in 189ms, sequenceid=234, compaction requested=true 2024-11-20T17:50:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:48,651 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T17:50:48,653 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 204226 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T17:50:48,653 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:48,653 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:48,653 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/15f8c571b4784d9aa7def42d02fa1660, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=199.4 K 2024-11-20T17:50:48,653 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:48,653 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/15f8c571b4784d9aa7def42d02fa1660, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4] 2024-11-20T17:50:48,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:48,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:48,654 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f8c571b4784d9aa7def42d02fa1660, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732125040046 2024-11-20T17:50:48,654 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864b48ae3ffa4819a44cb8345006d75c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732125042215 2024-11-20T17:50:48,654 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c003b98d0684769b250dcd72364a4bb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732125042868 2024-11-20T17:50:48,655 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac5ff601dbc94148b158d7b910acd6c9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732125043519 2024-11-20T17:50:48,655 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5434d5167ff4e3897283e636b91be65, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732125045700 2024-11-20T17:50:48,655 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T17:50:48,656 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb58e096fe445a7b4f05f31b8a4c8a4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732125047802 2024-11-20T17:50:48,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:48,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:48,658 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T17:50:48,658 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:48,658 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:48,658 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3f4a0d6e33024d699d564ccfdfc23233, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/9bebc37af26f4945b96c819712fdfd61, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d7648183ebf64ca8a7394aad92c50b9c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/743f681703a34f38ae2de6880a78229c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/5e71a118304a4281bebe5ecde59e1d55, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d2efbc13ff924947b31dd61f301c5218] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=71.5 K 2024-11-20T17:50:48,658 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4a0d6e33024d699d564ccfdfc23233, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732125040046 2024-11-20T17:50:48,659 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bebc37af26f4945b96c819712fdfd61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732125042215 2024-11-20T17:50:48,659 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d7648183ebf64ca8a7394aad92c50b9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732125042868 2024-11-20T17:50:48,659 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 743f681703a34f38ae2de6880a78229c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732125043519 2024-11-20T17:50:48,660 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e71a118304a4281bebe5ecde59e1d55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732125045700 2024-11-20T17:50:48,660 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d2efbc13ff924947b31dd61f301c5218, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732125047802 2024-11-20T17:50:48,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:48,678 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:48,682 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120dcfe949f4b6b45ffb3d504dc9b9cfa7d_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:48,684 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#549 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:48,685 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1a82e840bd3448edbdd5cc64abc38537 is 50, key is test_row_0/B:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,685 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120dcfe949f4b6b45ffb3d504dc9b9cfa7d_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:48,685 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dcfe949f4b6b45ffb3d504dc9b9cfa7d_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:48,731 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:48,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T17:50:48,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742477_1653 (size=12663) 2024-11-20T17:50:48,742 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1a82e840bd3448edbdd5cc64abc38537 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1a82e840bd3448edbdd5cc64abc38537 2024-11-20T17:50:48,748 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into 1a82e840bd3448edbdd5cc64abc38537(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:48,748 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:48,748 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=10, startTime=1732125048655; duration=0sec 2024-11-20T17:50:48,748 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:48,748 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:48,748 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-20T17:50:48,750 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-20T17:50:48,750 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:48,750 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:48,750 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/649a5dab3c2247fca5480bba7cf1a71d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b1882bc024284f59ba2063f3f6012b35, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/ff9a8e823d0d4dcf9bdf53e46d089d86, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f8767e92620b42539267d3c5fa14f111, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b5ed540c4f194537b6633d61c8a1f081, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f125a5ed58e34d00baa8fb6595afdaa9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=71.5 K 2024-11-20T17:50:48,751 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 649a5dab3c2247fca5480bba7cf1a71d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732125040046 2024-11-20T17:50:48,751 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b1882bc024284f59ba2063f3f6012b35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732125042215 2024-11-20T17:50:48,752 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9a8e823d0d4dcf9bdf53e46d089d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732125042868 2024-11-20T17:50:48,752 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f8767e92620b42539267d3c5fa14f111, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732125043519 2024-11-20T17:50:48,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742476_1652 (size=4469) 2024-11-20T17:50:48,753 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ed540c4f194537b6633d61c8a1f081, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732125045700 2024-11-20T17:50:48,753 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#548 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:48,753 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting f125a5ed58e34d00baa8fb6595afdaa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732125047802 2024-11-20T17:50:48,754 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/913fdf3f19a14abdbe4e1c60f90e3bca is 175, key is test_row_0/A:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,779 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#550 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:48,780 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/71ce2c2429a84b6897e435406fbe358a is 50, key is test_row_0/C:col10/1732125048460/Put/seqid=0 2024-11-20T17:50:48,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112021096e2549c4440ba567bf604447fe40_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125048543/Put/seqid=0 2024-11-20T17:50:48,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742478_1654 (size=31617) 2024-11-20T17:50:48,789 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/913fdf3f19a14abdbe4e1c60f90e3bca as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/913fdf3f19a14abdbe4e1c60f90e3bca 2024-11-20T17:50:48,794 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into 913fdf3f19a14abdbe4e1c60f90e3bca(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:48,794 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:48,794 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=10, startTime=1732125048651; duration=0sec 2024-11-20T17:50:48,794 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:48,794 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:50:48,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742480_1656 (size=14794) 2024-11-20T17:50:48,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742479_1655 (size=12663) 2024-11-20T17:50:48,831 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/71ce2c2429a84b6897e435406fbe358a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/71ce2c2429a84b6897e435406fbe358a 2024-11-20T17:50:48,836 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into 71ce2c2429a84b6897e435406fbe358a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:48,836 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:48,836 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=10, startTime=1732125048657; duration=0sec 2024-11-20T17:50:48,836 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:48,836 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:48,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125108837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125108947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125109157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:49,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112021096e2549c4440ba567bf604447fe40_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112021096e2549c4440ba567bf604447fe40_13207a781081472a057b14b506529437 2024-11-20T17:50:49,221 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6b24dbaebe054f3b8d3f06213b87da2f, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:49,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6b24dbaebe054f3b8d3f06213b87da2f is 175, key is test_row_0/A:col10/1732125048543/Put/seqid=0 2024-11-20T17:50:49,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742481_1657 (size=39749) 2024-11-20T17:50:49,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6b24dbaebe054f3b8d3f06213b87da2f 2024-11-20T17:50:49,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/b29ba127ab5a464dafee97930398e758 is 50, key is test_row_0/B:col10/1732125048543/Put/seqid=0 2024-11-20T17:50:49,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742482_1658 (size=12151) 2024-11-20T17:50:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T17:50:49,418 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:49,418 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-20T17:50:49,420 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T17:50:49,422 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:49,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:49,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125109467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T17:50:49,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T17:50:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:49,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T17:50:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:49,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T17:50:49,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/b29ba127ab5a464dafee97930398e758 2024-11-20T17:50:49,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/5b0db8148bcd43dbb34ec4a42e610d04 is 50, key is test_row_0/C:col10/1732125048543/Put/seqid=0 2024-11-20T17:50:49,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742483_1659 (size=12151) 2024-11-20T17:50:49,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T17:50:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125109877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125109877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,880 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125109877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,880 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,880 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125109886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,889 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:49,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125109977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:50,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T17:50:50,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:50,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:50,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:50,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T17:50:50,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/5b0db8148bcd43dbb34ec4a42e610d04 2024-11-20T17:50:50,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/6b24dbaebe054f3b8d3f06213b87da2f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f 2024-11-20T17:50:50,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f, entries=200, sequenceid=253, filesize=38.8 K 2024-11-20T17:50:50,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/b29ba127ab5a464dafee97930398e758 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/b29ba127ab5a464dafee97930398e758 2024-11-20T17:50:50,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/b29ba127ab5a464dafee97930398e758, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T17:50:50,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/5b0db8148bcd43dbb34ec4a42e610d04 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5b0db8148bcd43dbb34ec4a42e610d04 2024-11-20T17:50:50,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5b0db8148bcd43dbb34ec4a42e610d04, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T17:50:50,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 13207a781081472a057b14b506529437 in 1437ms, sequenceid=253, compaction requested=false 2024-11-20T17:50:50,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:50,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T17:50:50,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:50,185 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T17:50:50,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205e16e0eb391d411d9200038443249e3c_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125048835/Put/seqid=0 2024-11-20T17:50:50,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742484_1660 (size=12454) 2024-11-20T17:50:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T17:50:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,598 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205e16e0eb391d411d9200038443249e3c_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205e16e0eb391d411d9200038443249e3c_13207a781081472a057b14b506529437 2024-11-20T17:50:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f92c0842011945e986d66fae8f2ce2c4, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f92c0842011945e986d66fae8f2ce2c4 is 175, key is test_row_0/A:col10/1732125048835/Put/seqid=0 2024-11-20T17:50:50,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742485_1661 (size=31255) 2024-11-20T17:50:50,634 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f92c0842011945e986d66fae8f2ce2c4 2024-11-20T17:50:50,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/243c008c2757447ab6a47e3738f29e4a is 50, key is test_row_0/B:col10/1732125048835/Put/seqid=0 2024-11-20T17:50:50,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742486_1662 (size=12301) 2024-11-20T17:50:50,686 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/243c008c2757447ab6a47e3738f29e4a 2024-11-20T17:50:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b138bf25f01a456799abb0cfa8283ae1 is 50, key is test_row_0/C:col10/1732125048835/Put/seqid=0 2024-11-20T17:50:50,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742487_1663 (size=12301) 2024-11-20T17:50:50,711 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b138bf25f01a456799abb0cfa8283ae1 2024-11-20T17:50:50,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/f92c0842011945e986d66fae8f2ce2c4 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4 2024-11-20T17:50:50,722 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4, entries=150, sequenceid=273, filesize=30.5 K 2024-11-20T17:50:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/243c008c2757447ab6a47e3738f29e4a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/243c008c2757447ab6a47e3738f29e4a 2024-11-20T17:50:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,732 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/243c008c2757447ab6a47e3738f29e4a, entries=150, sequenceid=273, filesize=12.0 K 2024-11-20T17:50:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/b138bf25f01a456799abb0cfa8283ae1 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b138bf25f01a456799abb0cfa8283ae1 2024-11-20T17:50:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,737 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b138bf25f01a456799abb0cfa8283ae1, entries=150, sequenceid=273, filesize=12.0 K 2024-11-20T17:50:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,738 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 13207a781081472a057b14b506529437 in 553ms, sequenceid=273, compaction requested=true 2024-11-20T17:50:50,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:50,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:50,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-20T17:50:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-20T17:50:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-20T17:50:50,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3180 sec 2024-11-20T17:50:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.3240 sec 2024-11-20T17:50:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:51,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:51,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T17:50:51,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112027455a305b074953a097a30db141e3fb_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:51,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742488_1664 (size=14994) 2024-11-20T17:50:51,178 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:50:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125111234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:51,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125111347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:51,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:51,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:51,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125111557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:51,561 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112027455a305b074953a097a30db141e3fb_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027455a305b074953a097a30db141e3fb_13207a781081472a057b14b506529437 2024-11-20T17:50:51,562 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b2b82b8230ec46fb977e85cdade56898, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:51,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b2b82b8230ec46fb977e85cdade56898 is 175, key is test_row_0/A:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:51,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742489_1665 (size=39949) 2024-11-20T17:50:51,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b2b82b8230ec46fb977e85cdade56898 2024-11-20T17:50:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T17:50:51,567 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:51,569 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-20T17:50:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T17:50:51,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:51,572 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:51,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:51,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1aeea4a07d0b4bbebaa79250aea1da22 is 50, key is test_row_0/B:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:51,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742490_1666 (size=12301) 2024-11-20T17:50:51,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1aeea4a07d0b4bbebaa79250aea1da22 2024-11-20T17:50:51,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2cab03bd02ab47f8a8ba52f7ca995c96 is 50, key is test_row_0/C:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742491_1667 (size=12301) 2024-11-20T17:50:51,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2cab03bd02ab47f8a8ba52f7ca995c96 2024-11-20T17:50:51,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/b2b82b8230ec46fb977e85cdade56898 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898 2024-11-20T17:50:51,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898, entries=200, sequenceid=284, filesize=39.0 K 2024-11-20T17:50:51,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/1aeea4a07d0b4bbebaa79250aea1da22 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1aeea4a07d0b4bbebaa79250aea1da22 2024-11-20T17:50:51,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1aeea4a07d0b4bbebaa79250aea1da22, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T17:50:51,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2cab03bd02ab47f8a8ba52f7ca995c96 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2cab03bd02ab47f8a8ba52f7ca995c96 2024-11-20T17:50:51,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2cab03bd02ab47f8a8ba52f7ca995c96, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T17:50:51,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 13207a781081472a057b14b506529437 in 492ms, sequenceid=284, compaction requested=true 2024-11-20T17:50:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:51,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:51,626 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:50:51,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:51,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:51,626 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:50:51,628 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:50:51,628 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:51,628 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:51,628 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1a82e840bd3448edbdd5cc64abc38537, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/b29ba127ab5a464dafee97930398e758, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/243c008c2757447ab6a47e3738f29e4a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1aeea4a07d0b4bbebaa79250aea1da22] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=48.3 K 2024-11-20T17:50:51,629 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142570 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:50:51,629 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:51,629 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:51,629 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/913fdf3f19a14abdbe4e1c60f90e3bca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=139.2 K 2024-11-20T17:50:51,629 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:51,629 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/913fdf3f19a14abdbe4e1c60f90e3bca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898] 2024-11-20T17:50:51,630 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913fdf3f19a14abdbe4e1c60f90e3bca, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732125047802 2024-11-20T17:50:51,630 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a82e840bd3448edbdd5cc64abc38537, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732125047802 2024-11-20T17:50:51,631 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b24dbaebe054f3b8d3f06213b87da2f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732125048543 2024-11-20T17:50:51,631 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b29ba127ab5a464dafee97930398e758, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732125048543 2024-11-20T17:50:51,631 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f92c0842011945e986d66fae8f2ce2c4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732125048806 2024-11-20T17:50:51,631 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 243c008c2757447ab6a47e3738f29e4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732125048806 2024-11-20T17:50:51,631 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2b82b8230ec46fb977e85cdade56898, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732125051106 2024-11-20T17:50:51,631 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aeea4a07d0b4bbebaa79250aea1da22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732125051106 2024-11-20T17:50:51,641 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:51,649 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411201cf4e31970374e09819ca429b2bec5c2_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:51,650 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#561 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:51,651 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/75a69e033b1048a0a1946ab677c1ff61 is 50, key is test_row_0/B:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:51,652 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411201cf4e31970374e09819ca429b2bec5c2_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:51,652 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201cf4e31970374e09819ca429b2bec5c2_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T17:50:51,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742493_1669 (size=4469) 2024-11-20T17:50:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742492_1668 (size=12949) 2024-11-20T17:50:51,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T17:50:51,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:51,725 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T17:50:51,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bd8513cc592b4efbb2c43a2a730f3c19_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125051228/Put/seqid=0 2024-11-20T17:50:51,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742494_1670 (size=12454) 2024-11-20T17:50:51,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:51,758 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bd8513cc592b4efbb2c43a2a730f3c19_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bd8513cc592b4efbb2c43a2a730f3c19_13207a781081472a057b14b506529437 2024-11-20T17:50:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/3b0e8462efca4935851fc152f6592079, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/3b0e8462efca4935851fc152f6592079 is 175, key is test_row_0/A:col10/1732125051228/Put/seqid=0 2024-11-20T17:50:51,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742495_1671 (size=31255) 2024-11-20T17:50:51,762 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/3b0e8462efca4935851fc152f6592079 2024-11-20T17:50:51,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/8c4a57de3a8e4de79ec98d7a6a8c0889 is 50, key is test_row_0/B:col10/1732125051228/Put/seqid=0 2024-11-20T17:50:51,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742496_1672 (size=12301) 2024-11-20T17:50:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:51,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T17:50:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125111898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:52,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125112007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:52,086 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#560 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:52,087 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/78a58bd2eace4aeead4cb462dbe10629 is 175, key is test_row_0/A:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:52,095 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/75a69e033b1048a0a1946ab677c1ff61 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/75a69e033b1048a0a1946ab677c1ff61 2024-11-20T17:50:52,099 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into 75a69e033b1048a0a1946ab677c1ff61(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:52,099 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:52,099 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=12, startTime=1732125051625; duration=0sec 2024-11-20T17:50:52,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742497_1673 (size=31903) 2024-11-20T17:50:52,100 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:52,100 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:52,100 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T17:50:52,101 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T17:50:52,101 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:52,101 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:52,101 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/71ce2c2429a84b6897e435406fbe358a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5b0db8148bcd43dbb34ec4a42e610d04, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b138bf25f01a456799abb0cfa8283ae1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2cab03bd02ab47f8a8ba52f7ca995c96] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=48.3 K 2024-11-20T17:50:52,102 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ce2c2429a84b6897e435406fbe358a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732125047802 2024-11-20T17:50:52,102 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b0db8148bcd43dbb34ec4a42e610d04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732125048543 2024-11-20T17:50:52,102 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting b138bf25f01a456799abb0cfa8283ae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732125048806 2024-11-20T17:50:52,103 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cab03bd02ab47f8a8ba52f7ca995c96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732125051106 2024-11-20T17:50:52,110 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/78a58bd2eace4aeead4cb462dbe10629 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/78a58bd2eace4aeead4cb462dbe10629 2024-11-20T17:50:52,115 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into 78a58bd2eace4aeead4cb462dbe10629(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:52,115 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:52,116 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=12, startTime=1732125051625; duration=0sec 2024-11-20T17:50:52,116 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:52,116 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:50:52,116 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#564 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:52,116 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/fa1031b6e25c46ca9774e08c82982615 is 50, key is test_row_0/C:col10/1732125051132/Put/seqid=0 2024-11-20T17:50:52,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742498_1674 (size=12949) 2024-11-20T17:50:52,127 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/fa1031b6e25c46ca9774e08c82982615 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/fa1031b6e25c46ca9774e08c82982615 2024-11-20T17:50:52,132 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into fa1031b6e25c46ca9774e08c82982615(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:52,132 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:52,132 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=12, startTime=1732125051626; duration=0sec 2024-11-20T17:50:52,132 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:52,132 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:52,178 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/8c4a57de3a8e4de79ec98d7a6a8c0889 2024-11-20T17:50:52,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2f40477351ba464285452852e30151f5 is 50, key is test_row_0/C:col10/1732125051228/Put/seqid=0 2024-11-20T17:50:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T17:50:52,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742499_1675 (size=12301) 2024-11-20T17:50:52,205 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2f40477351ba464285452852e30151f5 2024-11-20T17:50:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/3b0e8462efca4935851fc152f6592079 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079 2024-11-20T17:50:52,216 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079, entries=150, sequenceid=309, filesize=30.5 K 2024-11-20T17:50:52,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/8c4a57de3a8e4de79ec98d7a6a8c0889 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/8c4a57de3a8e4de79ec98d7a6a8c0889 2024-11-20T17:50:52,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125112220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:52,232 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/8c4a57de3a8e4de79ec98d7a6a8c0889, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T17:50:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2f40477351ba464285452852e30151f5 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2f40477351ba464285452852e30151f5 2024-11-20T17:50:52,241 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2f40477351ba464285452852e30151f5, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T17:50:52,244 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 13207a781081472a057b14b506529437 in 519ms, sequenceid=309, compaction requested=false 2024-11-20T17:50:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:52,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-20T17:50:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-20T17:50:52,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-20T17:50:52,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 673 msec 2024-11-20T17:50:52,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 678 msec 2024-11-20T17:50:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:52,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T17:50:52,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201115d83353e54de286c0ea5dc6722d4a_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:52,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742500_1676 (size=14994) 2024-11-20T17:50:52,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:52,563 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201115d83353e54de286c0ea5dc6722d4a_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201115d83353e54de286c0ea5dc6722d4a_13207a781081472a057b14b506529437 2024-11-20T17:50:52,564 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/fec35c3e58bf413aa9f34b2491d5b53c, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:52,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/fec35c3e58bf413aa9f34b2491d5b53c is 175, key is test_row_0/A:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:52,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742501_1677 (size=39949) 2024-11-20T17:50:52,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125112623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T17:50:52,707 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:52,707 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T17:50:52,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-20T17:50:52,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T17:50:52,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T17:50:52,709 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T17:50:52,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T17:50:52,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125112737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T17:50:52,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T17:50:52,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:52,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:52,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:52,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125112947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:52,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/fec35c3e58bf413aa9f34b2491d5b53c 2024-11-20T17:50:52,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d4961d9abaad4e718045f0e5eae0861c is 50, key is test_row_0/B:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:52,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742502_1678 (size=12301) 2024-11-20T17:50:53,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d4961d9abaad4e718045f0e5eae0861c 2024-11-20T17:50:53,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/10e37561bce04562a13795bfea474050 is 50, key is test_row_0/C:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:53,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T17:50:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:50:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T17:50:53,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742503_1679 (size=12301) 2024-11-20T17:50:53,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/10e37561bce04562a13795bfea474050 2024-11-20T17:50:53,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/fec35c3e58bf413aa9f34b2491d5b53c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c 2024-11-20T17:50:53,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c, entries=200, sequenceid=324, filesize=39.0 K 2024-11-20T17:50:53,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/d4961d9abaad4e718045f0e5eae0861c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d4961d9abaad4e718045f0e5eae0861c 2024-11-20T17:50:53,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d4961d9abaad4e718045f0e5eae0861c, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T17:50:53,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/10e37561bce04562a13795bfea474050 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/10e37561bce04562a13795bfea474050 2024-11-20T17:50:53,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/10e37561bce04562a13795bfea474050, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T17:50:53,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 13207a781081472a057b14b506529437 in 529ms, sequenceid=324, compaction requested=true 2024-11-20T17:50:53,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:53,059 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:53,059 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:53,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:53,059 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:53,060 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:53,060 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,060 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/75a69e033b1048a0a1946ab677c1ff61, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/8c4a57de3a8e4de79ec98d7a6a8c0889, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d4961d9abaad4e718045f0e5eae0861c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=36.7 K 2024-11-20T17:50:53,060 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:53,060 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:53,060 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,060 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/78a58bd2eace4aeead4cb462dbe10629, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=100.7 K 2024-11-20T17:50:53,060 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,060 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/78a58bd2eace4aeead4cb462dbe10629, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c] 2024-11-20T17:50:53,060 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 75a69e033b1048a0a1946ab677c1ff61, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732125051106 2024-11-20T17:50:53,062 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78a58bd2eace4aeead4cb462dbe10629, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732125051106 2024-11-20T17:50:53,062 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c4a57de3a8e4de79ec98d7a6a8c0889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732125051216 2024-11-20T17:50:53,062 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0e8462efca4935851fc152f6592079, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732125051216 2024-11-20T17:50:53,062 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting d4961d9abaad4e718045f0e5eae0861c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732125051879 2024-11-20T17:50:53,062 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec35c3e58bf413aa9f34b2491d5b53c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732125051879 2024-11-20T17:50:53,070 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#569 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:53,070 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/ac81bebdc8cc4e508c341864a0f3cad6 is 50, key is test_row_0/B:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:53,077 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:53,101 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120e302530560374d62bcf6df48bda7b794_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:53,103 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120e302530560374d62bcf6df48bda7b794_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:53,104 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e302530560374d62bcf6df48bda7b794_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:53,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742504_1680 (size=13051) 2024-11-20T17:50:53,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742505_1681 (size=4469) 2024-11-20T17:50:53,166 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/ac81bebdc8cc4e508c341864a0f3cad6 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ac81bebdc8cc4e508c341864a0f3cad6 2024-11-20T17:50:53,168 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#570 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:53,168 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/5af1ddd63f3142a0b1f58de51454c478 is 175, key is test_row_0/A:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:53,173 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into ac81bebdc8cc4e508c341864a0f3cad6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:53,173 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:53,173 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=13, startTime=1732125053059; duration=0sec 2024-11-20T17:50:53,174 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:53,174 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:53,174 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:53,174 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:53,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T17:50:53,174 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,175 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T17:50:53,175 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:53,175 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/fa1031b6e25c46ca9774e08c82982615, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2f40477351ba464285452852e30151f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/10e37561bce04562a13795bfea474050] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=36.7 K 2024-11-20T17:50:53,176 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1031b6e25c46ca9774e08c82982615, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732125051106 2024-11-20T17:50:53,176 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f40477351ba464285452852e30151f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732125051216 2024-11-20T17:50:53,176 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e37561bce04562a13795bfea474050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732125051879 2024-11-20T17:50:53,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742506_1682 (size=32005) 2024-11-20T17:50:53,206 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#571 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:53,207 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/75fa5b6927e94a8faa9cbfb03cacda00 is 50, key is test_row_0/C:col10/1732125051891/Put/seqid=0 2024-11-20T17:50:53,208 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/5af1ddd63f3142a0b1f58de51454c478 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/5af1ddd63f3142a0b1f58de51454c478 2024-11-20T17:50:53,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f44d1aa58bac4067bc3f2f84bb1234ac_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125052587/Put/seqid=0 2024-11-20T17:50:53,215 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into 5af1ddd63f3142a0b1f58de51454c478(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:53,215 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:53,215 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=13, startTime=1732125053059; duration=0sec 2024-11-20T17:50:53,215 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:53,215 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:50:53,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742507_1683 (size=13051) 2024-11-20T17:50:53,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742508_1684 (size=12454) 2024-11-20T17:50:53,251 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/75fa5b6927e94a8faa9cbfb03cacda00 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/75fa5b6927e94a8faa9cbfb03cacda00 2024-11-20T17:50:53,256 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into 75fa5b6927e94a8faa9cbfb03cacda00(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:53,256 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:53,256 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=13, startTime=1732125053059; duration=0sec 2024-11-20T17:50:53,256 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:53,256 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:53,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. as already flushing 2024-11-20T17:50:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125113321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T17:50:53,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125113437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125113647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:53,653 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f44d1aa58bac4067bc3f2f84bb1234ac_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f44d1aa58bac4067bc3f2f84bb1234ac_13207a781081472a057b14b506529437 2024-11-20T17:50:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/2dfedf6e6bc1458b993aa4711c31bfbb, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/2dfedf6e6bc1458b993aa4711c31bfbb is 175, key is test_row_0/A:col10/1732125052587/Put/seqid=0 2024-11-20T17:50:53,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742509_1685 (size=31255) 2024-11-20T17:50:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T17:50:53,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35394 deadline: 1732125113898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,901 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_2' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35340 deadline: 1732125113927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35370 deadline: 1732125113927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1732125113927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,929 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,930 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_0' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,930 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 613581fcdc40,44957,1732124872893 for 'test_row_1' in 13207a781081472a057b14b506529437 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T17:50:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125113957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:54,067 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/2dfedf6e6bc1458b993aa4711c31bfbb 2024-11-20T17:50:54,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/7151c934db24456997da8b461124b65d is 50, key is test_row_0/B:col10/1732125052587/Put/seqid=0 2024-11-20T17:50:54,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742510_1686 (size=12301) 2024-11-20T17:50:54,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T17:50:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44957 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35352 deadline: 1732125114467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 2024-11-20T17:50:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 , the old value is region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:54,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T17:50:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., hostname=613581fcdc40,44957,1732124872893, seqNum=5 because the exception is null or not the one we care about 2024-11-20T17:50:54,476 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/7151c934db24456997da8b461124b65d 2024-11-20T17:50:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/15c8798c8619453abd6bf3a73ef2f5ac is 50, key is test_row_0/C:col10/1732125052587/Put/seqid=0 2024-11-20T17:50:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742511_1687 (size=12301) 2024-11-20T17:50:54,494 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/15c8798c8619453abd6bf3a73ef2f5ac 2024-11-20T17:50:54,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/2dfedf6e6bc1458b993aa4711c31bfbb as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb 2024-11-20T17:50:54,502 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb, entries=150, sequenceid=349, filesize=30.5 K 2024-11-20T17:50:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/7151c934db24456997da8b461124b65d as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/7151c934db24456997da8b461124b65d 2024-11-20T17:50:54,506 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/7151c934db24456997da8b461124b65d, entries=150, sequenceid=349, filesize=12.0 K 2024-11-20T17:50:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/15c8798c8619453abd6bf3a73ef2f5ac as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/15c8798c8619453abd6bf3a73ef2f5ac 2024-11-20T17:50:54,510 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/15c8798c8619453abd6bf3a73ef2f5ac, entries=150, sequenceid=349, filesize=12.0 K 2024-11-20T17:50:54,511 INFO [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 13207a781081472a057b14b506529437 in 1335ms, sequenceid=349, compaction requested=false 2024-11-20T17:50:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-20T17:50:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-20T17:50:54,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-20T17:50:54,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-20T17:50:54,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.8090 sec 2024-11-20T17:50:54,644 INFO [Thread-2635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2635. 2024-11-20T17:50:54,644 INFO [Thread-2641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2641. 2024-11-20T17:50:54,645 DEBUG [Thread-2635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:54,645 DEBUG [Thread-2641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:54,645 DEBUG [Thread-2641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,645 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,645 DEBUG [Thread-2641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,645 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:54,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:54,646 INFO [Thread-2639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2639. 2024-11-20T17:50:54,646 DEBUG [Thread-2639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:54,646 DEBUG [Thread-2639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,646 DEBUG [Thread-2639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:54,647 INFO [Thread-2637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2637. 2024-11-20T17:50:54,647 DEBUG [Thread-2637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:54,647 DEBUG [Thread-2637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,647 INFO [Thread-2633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2633. 2024-11-20T17:50:54,647 DEBUG [Thread-2633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:54,647 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,647 DEBUG [Thread-2637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:54,647 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:54,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T17:50:54,858 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:50:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44957 {}] regionserver.HRegion(8855): Flush requested on 13207a781081472a057b14b506529437 2024-11-20T17:50:55,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T17:50:55,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T17:50:55,479 INFO [Thread-2622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2622. 2024-11-20T17:50:55,479 DEBUG [Thread-2622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:50:55,479 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:55,479 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:50:55,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:50:55,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112052bfa3914fe34513a44d924150c74266_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:55,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742512_1688 (size=12454) 2024-11-20T17:50:55,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:50:55,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112052bfa3914fe34513a44d924150c74266_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112052bfa3914fe34513a44d924150c74266_13207a781081472a057b14b506529437 2024-11-20T17:50:55,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/0b7526e0b3d14904a4e9d1773a353b70, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:55,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/0b7526e0b3d14904a4e9d1773a353b70 is 175, key is test_row_0/A:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:55,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742513_1689 (size=31255) 2024-11-20T17:50:56,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/0b7526e0b3d14904a4e9d1773a353b70 2024-11-20T17:50:56,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/ae23975b9670457f859f993f630bcbcf is 50, key is test_row_0/B:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:56,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742514_1690 (size=12301) 2024-11-20T17:50:56,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/ae23975b9670457f859f993f630bcbcf 2024-11-20T17:50:56,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/1164b0c71cc7426d9852dfc2a58d5c69 is 50, key is test_row_0/C:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742515_1691 (size=12301) 2024-11-20T17:50:57,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/1164b0c71cc7426d9852dfc2a58d5c69 2024-11-20T17:50:57,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/0b7526e0b3d14904a4e9d1773a353b70 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70 2024-11-20T17:50:57,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70, entries=150, sequenceid=364, filesize=30.5 K 2024-11-20T17:50:57,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/ae23975b9670457f859f993f630bcbcf as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ae23975b9670457f859f993f630bcbcf 2024-11-20T17:50:57,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ae23975b9670457f859f993f630bcbcf, entries=150, sequenceid=364, filesize=12.0 K 2024-11-20T17:50:57,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/1164b0c71cc7426d9852dfc2a58d5c69 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/1164b0c71cc7426d9852dfc2a58d5c69 2024-11-20T17:50:57,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/1164b0c71cc7426d9852dfc2a58d5c69, entries=150, sequenceid=364, filesize=12.0 K 2024-11-20T17:50:57,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 13207a781081472a057b14b506529437 in 1653ms, sequenceid=364, compaction requested=true 2024-11-20T17:50:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T17:50:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:57,133 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T17:50:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:57,133 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13207a781081472a057b14b506529437:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T17:50:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:57,133 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:57,133 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:57,133 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/A is initiating minor compaction (all files) 2024-11-20T17:50:57,133 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/B is initiating minor compaction (all files) 2024-11-20T17:50:57,134 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/A in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:57,134 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/B in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:57,134 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/5af1ddd63f3142a0b1f58de51454c478, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=92.3 K 2024-11-20T17:50:57,134 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ac81bebdc8cc4e508c341864a0f3cad6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/7151c934db24456997da8b461124b65d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ae23975b9670457f859f993f630bcbcf] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=36.8 K 2024-11-20T17:50:57,134 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:57,134 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. files: [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/5af1ddd63f3142a0b1f58de51454c478, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70] 2024-11-20T17:50:57,134 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ac81bebdc8cc4e508c341864a0f3cad6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732125051879 2024-11-20T17:50:57,134 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7151c934db24456997da8b461124b65d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732125052587 2024-11-20T17:50:57,134 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af1ddd63f3142a0b1f58de51454c478, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732125051879 2024-11-20T17:50:57,134 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting ae23975b9670457f859f993f630bcbcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732125053274 2024-11-20T17:50:57,134 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dfedf6e6bc1458b993aa4711c31bfbb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732125052587 2024-11-20T17:50:57,135 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b7526e0b3d14904a4e9d1773a353b70, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732125053274 2024-11-20T17:50:57,138 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:57,139 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#B#compaction#578 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:57,139 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/14934358e3fe4ce6959b7e5f582cc76e is 50, key is test_row_0/B:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:57,139 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209f6c617be99c4d548227ba1b80545783_13207a781081472a057b14b506529437 store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:57,149 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209f6c617be99c4d548227ba1b80545783_13207a781081472a057b14b506529437, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:57,149 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209f6c617be99c4d548227ba1b80545783_13207a781081472a057b14b506529437 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:50:57,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742516_1692 (size=13153) 2024-11-20T17:50:57,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742517_1693 (size=4469) 2024-11-20T17:50:57,160 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/14934358e3fe4ce6959b7e5f582cc76e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/14934358e3fe4ce6959b7e5f582cc76e 2024-11-20T17:50:57,163 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/B of 13207a781081472a057b14b506529437 into 14934358e3fe4ce6959b7e5f582cc76e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:57,163 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:57,163 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/B, priority=13, startTime=1732125057133; duration=0sec 2024-11-20T17:50:57,163 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T17:50:57,163 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:B 2024-11-20T17:50:57,163 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T17:50:57,164 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T17:50:57,164 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1541): 13207a781081472a057b14b506529437/C is initiating minor compaction (all files) 2024-11-20T17:50:57,164 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13207a781081472a057b14b506529437/C in TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:50:57,164 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/75fa5b6927e94a8faa9cbfb03cacda00, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/15c8798c8619453abd6bf3a73ef2f5ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/1164b0c71cc7426d9852dfc2a58d5c69] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp, totalSize=36.8 K 2024-11-20T17:50:57,164 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 75fa5b6927e94a8faa9cbfb03cacda00, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732125051879 2024-11-20T17:50:57,165 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c8798c8619453abd6bf3a73ef2f5ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732125052587 2024-11-20T17:50:57,165 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1164b0c71cc7426d9852dfc2a58d5c69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732125053274 2024-11-20T17:50:57,172 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#C#compaction#580 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:57,172 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/e01815cc9974471d99e439e1ec3cedfa is 50, key is test_row_0/C:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:57,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742518_1694 (size=13153) 2024-11-20T17:50:57,559 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13207a781081472a057b14b506529437#A#compaction#579 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T17:50:57,559 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/456ef1e1608547d1975e9ce15fb4af9a is 175, key is test_row_0/A:col10/1732125053274/Put/seqid=0 2024-11-20T17:50:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742519_1695 (size=32107) 2024-11-20T17:50:57,579 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/e01815cc9974471d99e439e1ec3cedfa as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/e01815cc9974471d99e439e1ec3cedfa 2024-11-20T17:50:57,583 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/C of 13207a781081472a057b14b506529437 into e01815cc9974471d99e439e1ec3cedfa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:57,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:57,583 INFO [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/C, priority=13, startTime=1732125057133; duration=0sec 2024-11-20T17:50:57,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:57,583 DEBUG [RS:0;613581fcdc40:44957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:C 2024-11-20T17:50:57,968 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/456ef1e1608547d1975e9ce15fb4af9a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/456ef1e1608547d1975e9ce15fb4af9a 2024-11-20T17:50:57,972 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13207a781081472a057b14b506529437/A of 13207a781081472a057b14b506529437 into 456ef1e1608547d1975e9ce15fb4af9a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T17:50:57,972 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13207a781081472a057b14b506529437: 2024-11-20T17:50:57,972 INFO [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., storeName=13207a781081472a057b14b506529437/A, priority=13, startTime=1732125057133; duration=0sec 2024-11-20T17:50:57,972 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T17:50:57,972 DEBUG [RS:0;613581fcdc40:44957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13207a781081472a057b14b506529437:A 2024-11-20T17:51:03,998 INFO [Thread-2628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2628. 2024-11-20T17:51:03,998 DEBUG [Thread-2628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:51:03,998 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:03,998 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:03,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:51:04,009 INFO [Thread-2626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2626. 2024-11-20T17:51:04,009 DEBUG [Thread-2626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:51:04,009 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,009 INFO [Thread-2630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2630. 2024-11-20T17:51:04,009 INFO [Thread-2624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2624. 2024-11-20T17:51:04,009 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,009 DEBUG [Thread-2630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:51:04,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:51:04,009 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,009 DEBUG [Thread-2624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T17:51:04,009 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,009 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:51:04,009 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3415 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3430 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3354 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3414 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3447 2024-11-20T17:51:04,010 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T17:51:04,010 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:51:04,010 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:51:04,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:04,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:51:04,011 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T17:51:04,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T17:51:04,013 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125064013"}]},"ts":"1732125064013"} 2024-11-20T17:51:04,014 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T17:51:04,014 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T17:51:04,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T17:51:04,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, UNASSIGN}] 2024-11-20T17:51:04,017 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, UNASSIGN 2024-11-20T17:51:04,017 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=CLOSING, regionLocation=613581fcdc40,44957,1732124872893 2024-11-20T17:51:04,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, UNASSIGN because future has completed 2024-11-20T17:51:04,018 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T17:51:04,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893}] 2024-11-20T17:51:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T17:51:04,171 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 13207a781081472a057b14b506529437 2024-11-20T17:51:04,171 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T17:51:04,171 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 13207a781081472a057b14b506529437, disabling compactions & flushes 2024-11-20T17:51:04,171 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:51:04,171 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:51:04,171 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. after waiting 0 ms 2024-11-20T17:51:04,171 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:51:04,171 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 13207a781081472a057b14b506529437 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-20T17:51:04,175 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af0c0cc849c84b109a10d5e1d2c515c4_13207a781081472a057b14b506529437 is 50, key is test_row_0/A:col10/1732125064007/Put/seqid=0 2024-11-20T17:51:04,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742520_1696 (size=12454) 2024-11-20T17:51:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T17:51:04,579 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:51:04,583 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af0c0cc849c84b109a10d5e1d2c515c4_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af0c0cc849c84b109a10d5e1d2c515c4_13207a781081472a057b14b506529437 2024-11-20T17:51:04,584 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/83232e4d68344b9eb30c5f53e17d967a, store: [table=TestAcidGuarantees family=A region=13207a781081472a057b14b506529437] 2024-11-20T17:51:04,584 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/83232e4d68344b9eb30c5f53e17d967a is 175, key is test_row_0/A:col10/1732125064007/Put/seqid=0 2024-11-20T17:51:04,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742521_1697 (size=31255) 2024-11-20T17:51:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T17:51:04,995 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/83232e4d68344b9eb30c5f53e17d967a 2024-11-20T17:51:05,002 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/605d4e81d7ef4682a97d687271dfd40b is 50, key is test_row_0/B:col10/1732125064007/Put/seqid=0 2024-11-20T17:51:05,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742522_1698 (size=12301) 2024-11-20T17:51:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T17:51:05,405 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/605d4e81d7ef4682a97d687271dfd40b 2024-11-20T17:51:05,411 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2e4273c9f72a4df990b3d02c3b5c7d0c is 50, key is test_row_0/C:col10/1732125064007/Put/seqid=0 2024-11-20T17:51:05,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742523_1699 (size=12301) 2024-11-20T17:51:05,816 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2e4273c9f72a4df990b3d02c3b5c7d0c 2024-11-20T17:51:05,819 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/A/83232e4d68344b9eb30c5f53e17d967a as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/83232e4d68344b9eb30c5f53e17d967a 2024-11-20T17:51:05,822 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/83232e4d68344b9eb30c5f53e17d967a, entries=150, sequenceid=374, filesize=30.5 K 2024-11-20T17:51:05,822 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/B/605d4e81d7ef4682a97d687271dfd40b as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/605d4e81d7ef4682a97d687271dfd40b 2024-11-20T17:51:05,825 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/605d4e81d7ef4682a97d687271dfd40b, entries=150, sequenceid=374, filesize=12.0 K 2024-11-20T17:51:05,825 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/.tmp/C/2e4273c9f72a4df990b3d02c3b5c7d0c as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2e4273c9f72a4df990b3d02c3b5c7d0c 2024-11-20T17:51:05,828 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2e4273c9f72a4df990b3d02c3b5c7d0c, entries=150, sequenceid=374, filesize=12.0 K 2024-11-20T17:51:05,828 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 13207a781081472a057b14b506529437 in 1657ms, sequenceid=374, compaction requested=false 2024-11-20T17:51:05,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/dd672781248240408a1eb629a91bda89, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/faeba2a4e0494e31bead1e7f2176fb25, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/15f8c571b4784d9aa7def42d02fa1660, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/913fdf3f19a14abdbe4e1c60f90e3bca, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/78a58bd2eace4aeead4cb462dbe10629, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/5af1ddd63f3142a0b1f58de51454c478, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70] to archive 2024-11-20T17:51:05,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:51:05,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/cf1c10f8b1b149f7b265ee02935db368 2024-11-20T17:51:05,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/1fba23047b7a4f13b28958fb2a8d62b0 2024-11-20T17:51:05,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f5e676d0a777419c8adb3ce1cac322f9 2024-11-20T17:51:05,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/dd672781248240408a1eb629a91bda89 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/dd672781248240408a1eb629a91bda89 2024-11-20T17:51:05,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/66e383209d674045b4b817290baf3e37 2024-11-20T17:51:05,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/faeba2a4e0494e31bead1e7f2176fb25 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/faeba2a4e0494e31bead1e7f2176fb25 2024-11-20T17:51:05,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ebdc330d877a4976b7959e2b70a429e2 2024-11-20T17:51:05,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b4a56b134a2a426096ea4c8f7436b437 2024-11-20T17:51:05,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/15f8c571b4784d9aa7def42d02fa1660 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/15f8c571b4784d9aa7def42d02fa1660 2024-11-20T17:51:05,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/08b9e8cbf9ac4a88896059e48aeb57f6 2024-11-20T17:51:05,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/864b48ae3ffa4819a44cb8345006d75c 2024-11-20T17:51:05,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6c003b98d0684769b250dcd72364a4bb 2024-11-20T17:51:05,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/ac5ff601dbc94148b158d7b910acd6c9 2024-11-20T17:51:05,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/a5434d5167ff4e3897283e636b91be65 2024-11-20T17:51:05,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/bcb58e096fe445a7b4f05f31b8a4c8a4 2024-11-20T17:51:05,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/913fdf3f19a14abdbe4e1c60f90e3bca to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/913fdf3f19a14abdbe4e1c60f90e3bca 2024-11-20T17:51:05,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/6b24dbaebe054f3b8d3f06213b87da2f 2024-11-20T17:51:05,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/f92c0842011945e986d66fae8f2ce2c4 2024-11-20T17:51:05,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/b2b82b8230ec46fb977e85cdade56898 2024-11-20T17:51:05,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/78a58bd2eace4aeead4cb462dbe10629 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/78a58bd2eace4aeead4cb462dbe10629 2024-11-20T17:51:05,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/3b0e8462efca4935851fc152f6592079 2024-11-20T17:51:05,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/fec35c3e58bf413aa9f34b2491d5b53c 2024-11-20T17:51:05,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/5af1ddd63f3142a0b1f58de51454c478 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/5af1ddd63f3142a0b1f58de51454c478 2024-11-20T17:51:05,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/2dfedf6e6bc1458b993aa4711c31bfbb 2024-11-20T17:51:05,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/0b7526e0b3d14904a4e9d1773a353b70 2024-11-20T17:51:05,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1baf74d9fe7d4defa6ba5ad803938f83, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/bb4b8c0105fc475ab2dca6627b18b570, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3c3019e1098747eb981e0d9e85675b8f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/461975d1563a466e8f4b10dac60ce586, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/875c61102a764481b40222061f191547, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/96b50204f9504335a5276091b3327fcc, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d915217777f34c31b113b6d6d1af9dbf, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/e293943852564f38a9b07653f6e7fa55, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3f4a0d6e33024d699d564ccfdfc23233, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/48ebbe7ff3244760bb340c39f693e5ae, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/9bebc37af26f4945b96c819712fdfd61, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d7648183ebf64ca8a7394aad92c50b9c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/743f681703a34f38ae2de6880a78229c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/5e71a118304a4281bebe5ecde59e1d55, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1a82e840bd3448edbdd5cc64abc38537, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d2efbc13ff924947b31dd61f301c5218, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/b29ba127ab5a464dafee97930398e758, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/243c008c2757447ab6a47e3738f29e4a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/75a69e033b1048a0a1946ab677c1ff61, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1aeea4a07d0b4bbebaa79250aea1da22, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/8c4a57de3a8e4de79ec98d7a6a8c0889, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ac81bebdc8cc4e508c341864a0f3cad6, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d4961d9abaad4e718045f0e5eae0861c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/7151c934db24456997da8b461124b65d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ae23975b9670457f859f993f630bcbcf] to archive 2024-11-20T17:51:05,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:51:05,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1baf74d9fe7d4defa6ba5ad803938f83 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1baf74d9fe7d4defa6ba5ad803938f83 2024-11-20T17:51:05,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/bb4b8c0105fc475ab2dca6627b18b570 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/bb4b8c0105fc475ab2dca6627b18b570 2024-11-20T17:51:05,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3c3019e1098747eb981e0d9e85675b8f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3c3019e1098747eb981e0d9e85675b8f 2024-11-20T17:51:05,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/461975d1563a466e8f4b10dac60ce586 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/461975d1563a466e8f4b10dac60ce586 2024-11-20T17:51:05,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/875c61102a764481b40222061f191547 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/875c61102a764481b40222061f191547 2024-11-20T17:51:05,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/96b50204f9504335a5276091b3327fcc to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/96b50204f9504335a5276091b3327fcc 2024-11-20T17:51:05,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d915217777f34c31b113b6d6d1af9dbf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d915217777f34c31b113b6d6d1af9dbf 2024-11-20T17:51:05,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/e293943852564f38a9b07653f6e7fa55 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/e293943852564f38a9b07653f6e7fa55 2024-11-20T17:51:05,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3f4a0d6e33024d699d564ccfdfc23233 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/3f4a0d6e33024d699d564ccfdfc23233 2024-11-20T17:51:05,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/48ebbe7ff3244760bb340c39f693e5ae to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/48ebbe7ff3244760bb340c39f693e5ae 2024-11-20T17:51:05,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/9bebc37af26f4945b96c819712fdfd61 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/9bebc37af26f4945b96c819712fdfd61 2024-11-20T17:51:05,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d7648183ebf64ca8a7394aad92c50b9c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d7648183ebf64ca8a7394aad92c50b9c 2024-11-20T17:51:05,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/743f681703a34f38ae2de6880a78229c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/743f681703a34f38ae2de6880a78229c 2024-11-20T17:51:05,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/5e71a118304a4281bebe5ecde59e1d55 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/5e71a118304a4281bebe5ecde59e1d55 2024-11-20T17:51:05,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1a82e840bd3448edbdd5cc64abc38537 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1a82e840bd3448edbdd5cc64abc38537 2024-11-20T17:51:05,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d2efbc13ff924947b31dd61f301c5218 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d2efbc13ff924947b31dd61f301c5218 2024-11-20T17:51:05,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/b29ba127ab5a464dafee97930398e758 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/b29ba127ab5a464dafee97930398e758 2024-11-20T17:51:05,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/243c008c2757447ab6a47e3738f29e4a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/243c008c2757447ab6a47e3738f29e4a 2024-11-20T17:51:05,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/75a69e033b1048a0a1946ab677c1ff61 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/75a69e033b1048a0a1946ab677c1ff61 2024-11-20T17:51:05,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1aeea4a07d0b4bbebaa79250aea1da22 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/1aeea4a07d0b4bbebaa79250aea1da22 2024-11-20T17:51:05,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/8c4a57de3a8e4de79ec98d7a6a8c0889 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/8c4a57de3a8e4de79ec98d7a6a8c0889 2024-11-20T17:51:05,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ac81bebdc8cc4e508c341864a0f3cad6 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ac81bebdc8cc4e508c341864a0f3cad6 2024-11-20T17:51:05,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d4961d9abaad4e718045f0e5eae0861c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/d4961d9abaad4e718045f0e5eae0861c 2024-11-20T17:51:05,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/7151c934db24456997da8b461124b65d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/7151c934db24456997da8b461124b65d 2024-11-20T17:51:05,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ae23975b9670457f859f993f630bcbcf to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/ae23975b9670457f859f993f630bcbcf 2024-11-20T17:51:05,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/cc11f1c4b3754eefb594c5f6cacd5d69, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/7a6079d303c94068970ea5c48b99eddd, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/c0cd668aa3084cb79f26a968f3fc2299, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5613a279ae4241a9b0c9ce4e837fa264, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/98b6aa5132ec48628ebf321dc7aaf69f, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/8a9b3bb611f8496fa334664ae61ac232, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b84889fc0832430587d30bf4898de258, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/31de302da2b143e79fae5fb633ec373c, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/649a5dab3c2247fca5480bba7cf1a71d, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/41b0f8cab76c4c55b67f147a465c7705, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b1882bc024284f59ba2063f3f6012b35, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/ff9a8e823d0d4dcf9bdf53e46d089d86, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f8767e92620b42539267d3c5fa14f111, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b5ed540c4f194537b6633d61c8a1f081, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/71ce2c2429a84b6897e435406fbe358a, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f125a5ed58e34d00baa8fb6595afdaa9, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5b0db8148bcd43dbb34ec4a42e610d04, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b138bf25f01a456799abb0cfa8283ae1, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/fa1031b6e25c46ca9774e08c82982615, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2cab03bd02ab47f8a8ba52f7ca995c96, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2f40477351ba464285452852e30151f5, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/75fa5b6927e94a8faa9cbfb03cacda00, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/10e37561bce04562a13795bfea474050, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/15c8798c8619453abd6bf3a73ef2f5ac, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/1164b0c71cc7426d9852dfc2a58d5c69] to archive 2024-11-20T17:51:05,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T17:51:05,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/cc11f1c4b3754eefb594c5f6cacd5d69 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/cc11f1c4b3754eefb594c5f6cacd5d69 2024-11-20T17:51:05,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/7a6079d303c94068970ea5c48b99eddd to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/7a6079d303c94068970ea5c48b99eddd 2024-11-20T17:51:05,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/c0cd668aa3084cb79f26a968f3fc2299 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/c0cd668aa3084cb79f26a968f3fc2299 2024-11-20T17:51:05,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5613a279ae4241a9b0c9ce4e837fa264 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5613a279ae4241a9b0c9ce4e837fa264 2024-11-20T17:51:05,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/98b6aa5132ec48628ebf321dc7aaf69f to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/98b6aa5132ec48628ebf321dc7aaf69f 2024-11-20T17:51:05,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/8a9b3bb611f8496fa334664ae61ac232 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/8a9b3bb611f8496fa334664ae61ac232 2024-11-20T17:51:05,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b84889fc0832430587d30bf4898de258 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b84889fc0832430587d30bf4898de258 2024-11-20T17:51:05,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/31de302da2b143e79fae5fb633ec373c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/31de302da2b143e79fae5fb633ec373c 2024-11-20T17:51:05,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/649a5dab3c2247fca5480bba7cf1a71d to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/649a5dab3c2247fca5480bba7cf1a71d 2024-11-20T17:51:05,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/41b0f8cab76c4c55b67f147a465c7705 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/41b0f8cab76c4c55b67f147a465c7705 2024-11-20T17:51:05,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b1882bc024284f59ba2063f3f6012b35 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b1882bc024284f59ba2063f3f6012b35 2024-11-20T17:51:05,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/ff9a8e823d0d4dcf9bdf53e46d089d86 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/ff9a8e823d0d4dcf9bdf53e46d089d86 2024-11-20T17:51:05,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f8767e92620b42539267d3c5fa14f111 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f8767e92620b42539267d3c5fa14f111 2024-11-20T17:51:05,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b5ed540c4f194537b6633d61c8a1f081 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b5ed540c4f194537b6633d61c8a1f081 2024-11-20T17:51:05,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/71ce2c2429a84b6897e435406fbe358a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/71ce2c2429a84b6897e435406fbe358a 2024-11-20T17:51:05,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f125a5ed58e34d00baa8fb6595afdaa9 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/f125a5ed58e34d00baa8fb6595afdaa9 2024-11-20T17:51:05,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5b0db8148bcd43dbb34ec4a42e610d04 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/5b0db8148bcd43dbb34ec4a42e610d04 2024-11-20T17:51:05,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b138bf25f01a456799abb0cfa8283ae1 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/b138bf25f01a456799abb0cfa8283ae1 2024-11-20T17:51:05,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/fa1031b6e25c46ca9774e08c82982615 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/fa1031b6e25c46ca9774e08c82982615 2024-11-20T17:51:05,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2cab03bd02ab47f8a8ba52f7ca995c96 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2cab03bd02ab47f8a8ba52f7ca995c96 2024-11-20T17:51:05,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2f40477351ba464285452852e30151f5 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2f40477351ba464285452852e30151f5 2024-11-20T17:51:05,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/75fa5b6927e94a8faa9cbfb03cacda00 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/75fa5b6927e94a8faa9cbfb03cacda00 2024-11-20T17:51:05,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/10e37561bce04562a13795bfea474050 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/10e37561bce04562a13795bfea474050 2024-11-20T17:51:05,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/15c8798c8619453abd6bf3a73ef2f5ac to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/15c8798c8619453abd6bf3a73ef2f5ac 2024-11-20T17:51:05,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/1164b0c71cc7426d9852dfc2a58d5c69 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/1164b0c71cc7426d9852dfc2a58d5c69 2024-11-20T17:51:05,887 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=4 2024-11-20T17:51:05,888 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. 2024-11-20T17:51:05,888 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 13207a781081472a057b14b506529437: Waiting for close lock at 1732125064171Running coprocessor pre-close hooks at 1732125064171Disabling compacts and flushes for region at 1732125064171Disabling writes for close at 1732125064171Obtaining lock to block concurrent updates at 1732125064171Preparing flush snapshotting stores in 13207a781081472a057b14b506529437 at 1732125064171Finished memstore snapshotting TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732125064171Flushing stores of TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437. at 1732125064172 (+1 ms)Flushing 13207a781081472a057b14b506529437/A: creating writer at 1732125064172Flushing mob file 13207a781081472a057b14b506529437/A: appending metadata at 1732125064175 (+3 ms)Flushing mob file 13207a781081472a057b14b506529437/A: closing flushed file at 1732125064175Flushing 13207a781081472a057b14b506529437/A: appending metadata at 1732125064584 (+409 ms)Flushing 13207a781081472a057b14b506529437/A: closing flushed file at 1732125064584Flushing 13207a781081472a057b14b506529437/B: creating writer at 1732125064999 (+415 ms)Flushing 13207a781081472a057b14b506529437/B: appending metadata at 1732125065001 (+2 ms)Flushing 13207a781081472a057b14b506529437/B: closing flushed file at 1732125065001Flushing 13207a781081472a057b14b506529437/C: creating writer at 1732125065409 (+408 ms)Flushing 13207a781081472a057b14b506529437/C: appending metadata at 1732125065411 (+2 ms)Flushing 13207a781081472a057b14b506529437/C: closing flushed file at 1732125065411Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9502e43: reopening flushed file at 1732125065819 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37952662: reopening flushed file at 1732125065822 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7be8e799: reopening flushed file at 1732125065825 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 13207a781081472a057b14b506529437 in 1657ms, sequenceid=374, compaction requested=false at 1732125065828 (+3 ms)Writing region close event to WAL at 1732125065885 (+57 ms)Running coprocessor post-close hooks at 1732125065888 (+3 ms)Closed at 1732125065888 2024-11-20T17:51:05,890 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 13207a781081472a057b14b506529437 2024-11-20T17:51:05,890 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=13207a781081472a057b14b506529437, regionState=CLOSED 2024-11-20T17:51:05,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 because future has completed 2024-11-20T17:51:05,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-20T17:51:05,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 13207a781081472a057b14b506529437, server=613581fcdc40,44957,1732124872893 in 1.8740 sec 2024-11-20T17:51:05,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-20T17:51:05,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13207a781081472a057b14b506529437, UNASSIGN in 1.8770 sec 2024-11-20T17:51:05,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-20T17:51:05,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8790 sec 2024-11-20T17:51:05,896 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125065896"}]},"ts":"1732125065896"} 2024-11-20T17:51:05,897 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T17:51:05,897 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T17:51:05,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8860 sec 2024-11-20T17:51:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T17:51:06,157 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:51:06,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T17:51:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:06,160 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T17:51:06,160 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:06,163 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437 2024-11-20T17:51:06,165 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/recovered.edits] 2024-11-20T17:51:06,166 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/456ef1e1608547d1975e9ce15fb4af9a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/456ef1e1608547d1975e9ce15fb4af9a 2024-11-20T17:51:06,167 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/83232e4d68344b9eb30c5f53e17d967a to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/A/83232e4d68344b9eb30c5f53e17d967a 2024-11-20T17:51:06,168 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/14934358e3fe4ce6959b7e5f582cc76e to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/14934358e3fe4ce6959b7e5f582cc76e 2024-11-20T17:51:06,169 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/605d4e81d7ef4682a97d687271dfd40b to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/B/605d4e81d7ef4682a97d687271dfd40b 2024-11-20T17:51:06,170 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2e4273c9f72a4df990b3d02c3b5c7d0c to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/2e4273c9f72a4df990b3d02c3b5c7d0c 2024-11-20T17:51:06,171 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/e01815cc9974471d99e439e1ec3cedfa to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/C/e01815cc9974471d99e439e1ec3cedfa 2024-11-20T17:51:06,173 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/recovered.edits/377.seqid to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437/recovered.edits/377.seqid 2024-11-20T17:51:06,174 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/default/TestAcidGuarantees/13207a781081472a057b14b506529437 2024-11-20T17:51:06,174 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T17:51:06,174 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T17:51:06,175 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T17:51:06,178 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120076ecac078504f9894f44dcf20e03e20_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120076ecac078504f9894f44dcf20e03e20_13207a781081472a057b14b506529437 2024-11-20T17:51:06,179 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200d894785397e481188f7b314e87d749b_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200d894785397e481188f7b314e87d749b_13207a781081472a057b14b506529437 2024-11-20T17:51:06,181 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201115d83353e54de286c0ea5dc6722d4a_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201115d83353e54de286c0ea5dc6722d4a_13207a781081472a057b14b506529437 2024-11-20T17:51:06,182 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112021096e2549c4440ba567bf604447fe40_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112021096e2549c4440ba567bf604447fe40_13207a781081472a057b14b506529437 2024-11-20T17:51:06,183 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120264955628baf40339a24c002e55535ee_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120264955628baf40339a24c002e55535ee_13207a781081472a057b14b506529437 2024-11-20T17:51:06,187 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027455a305b074953a097a30db141e3fb_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112027455a305b074953a097a30db141e3fb_13207a781081472a057b14b506529437 2024-11-20T17:51:06,188 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203239120fc3a04df3aabd0c0f673bac63_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203239120fc3a04df3aabd0c0f673bac63_13207a781081472a057b14b506529437 2024-11-20T17:51:06,189 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112052bfa3914fe34513a44d924150c74266_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112052bfa3914fe34513a44d924150c74266_13207a781081472a057b14b506529437 2024-11-20T17:51:06,190 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205349ccce668a4df190f7999d6809b1f8_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205349ccce668a4df190f7999d6809b1f8_13207a781081472a057b14b506529437 2024-11-20T17:51:06,191 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205e16e0eb391d411d9200038443249e3c_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205e16e0eb391d411d9200038443249e3c_13207a781081472a057b14b506529437 2024-11-20T17:51:06,193 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112061582b8dff074642ba0db947957f58f0_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112061582b8dff074642ba0db947957f58f0_13207a781081472a057b14b506529437 2024-11-20T17:51:06,194 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112070e66cbd9975448385d47bdfbb6de9a0_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112070e66cbd9975448385d47bdfbb6de9a0_13207a781081472a057b14b506529437 2024-11-20T17:51:06,195 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120846597cbe20e43febf9ada058ccadb63_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120846597cbe20e43febf9ada058ccadb63_13207a781081472a057b14b506529437 2024-11-20T17:51:06,196 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e9f615866cb47aca8e4e5150ffbd116_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e9f615866cb47aca8e4e5150ffbd116_13207a781081472a057b14b506529437 2024-11-20T17:51:06,197 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120acf7c428ba5e457988a59a9c4b5adc20_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120acf7c428ba5e457988a59a9c4b5adc20_13207a781081472a057b14b506529437 2024-11-20T17:51:06,198 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af0c0cc849c84b109a10d5e1d2c515c4_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af0c0cc849c84b109a10d5e1d2c515c4_13207a781081472a057b14b506529437 2024-11-20T17:51:06,198 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bd8513cc592b4efbb2c43a2a730f3c19_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bd8513cc592b4efbb2c43a2a730f3c19_13207a781081472a057b14b506529437 2024-11-20T17:51:06,199 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d76b096d959840a38a34d275c46f5d70_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d76b096d959840a38a34d275c46f5d70_13207a781081472a057b14b506529437 2024-11-20T17:51:06,200 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f1ded8539559495d81681fa7f61250d4_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f1ded8539559495d81681fa7f61250d4_13207a781081472a057b14b506529437 2024-11-20T17:51:06,201 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f44d1aa58bac4067bc3f2f84bb1234ac_13207a781081472a057b14b506529437 to hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f44d1aa58bac4067bc3f2f84bb1234ac_13207a781081472a057b14b506529437 2024-11-20T17:51:06,202 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T17:51:06,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:06,206 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T17:51:06,208 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T17:51:06,210 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:06,210 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T17:51:06,210 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732125066210"}]},"ts":"9223372036854775807"} 2024-11-20T17:51:06,212 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T17:51:06,212 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 13207a781081472a057b14b506529437, NAME => 'TestAcidGuarantees,,1732125031518.13207a781081472a057b14b506529437.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T17:51:06,212 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T17:51:06,213 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732125066212"}]},"ts":"9223372036854775807"} 2024-11-20T17:51:06,214 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T17:51:06,214 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T17:51:06,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-20T17:51:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39775 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T17:51:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T17:51:06,268 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T17:51:06,276 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 241), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=561 (was 611), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=7901 (was 8389) 2024-11-20T17:51:06,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T17:51:06,276 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:51:06,276 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:51:06,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:06,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:06,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T17:51:06,277 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T17:51:06,277 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-11-20T17:51:06,277 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=613581fcdc40,39775,1732124872132 2024-11-20T17:51:06,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:51:06,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:51:06,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:51:06,352 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T17:51:06,352 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:51:06,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:06,352 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:51:06,353 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,44957,1732124872893' ***** 2024-11-20T17:51:06,353 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T17:51:06,353 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T17:51:06,353 INFO [RS:0;613581fcdc40:44957 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T17:51:06,353 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T17:51:06,353 INFO [RS:0;613581fcdc40:44957 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T17:51:06,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T17:51:06,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,44957,1732124872893 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;613581fcdc40:44957. 2024-11-20T17:51:06,354 DEBUG [RS:0;613581fcdc40:44957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T17:51:06,354 DEBUG [RS:0;613581fcdc40:44957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T17:51:06,354 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T17:51:06,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:51:06,355 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T17:51:06,355 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T17:51:06,355 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:06,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:51:06,355 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:51:06,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:51:06,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:51:06,355 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:51:06,355 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T17:51:06,379 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/info/30eabf6e32ae4b3fbaa7a86bf4796c24 is 95, key is TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8./info:/1732124907457/DeleteFamily/seqid=0 2024-11-20T17:51:06,385 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:51:06,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742524_1700 (size=6514) 2024-11-20T17:51:06,555 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:06,712 INFO [regionserver/613581fcdc40:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T17:51:06,712 INFO [regionserver/613581fcdc40:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T17:51:06,755 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:06,793 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/info/30eabf6e32ae4b3fbaa7a86bf4796c24 2024-11-20T17:51:06,811 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/ns/435491b806024e379ac96b4ec30c1312 is 93, key is TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8./ns:/1732124907416/DeleteFamily/seqid=0 2024-11-20T17:51:06,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742525_1701 (size=5906) 2024-11-20T17:51:06,956 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:07,156 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:07,215 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/ns/435491b806024e379ac96b4ec30c1312 2024-11-20T17:51:07,242 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/rep_barrier/4b6e4dbfed774f6d909261a464e4a355 is 102, key is TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8./rep_barrier:/1732124907416/DeleteFamily/seqid=0 2024-11-20T17:51:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742526_1702 (size=6025) 2024-11-20T17:51:07,356 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:07,556 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:07,646 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/rep_barrier/4b6e4dbfed774f6d909261a464e4a355 2024-11-20T17:51:07,666 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/table/fc441ded672d47cbb410d05dd5413c2f is 96, key is TestAcidGuarantees,,1732124877191.f6561bfd37132223c1b60b9e3a6231d8./table:/1732124907416/DeleteFamily/seqid=0 2024-11-20T17:51:07,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742527_1703 (size=5936) 2024-11-20T17:51:07,757 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:07,957 DEBUG [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T17:51:08,070 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/table/fc441ded672d47cbb410d05dd5413c2f 2024-11-20T17:51:08,073 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/info/30eabf6e32ae4b3fbaa7a86bf4796c24 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/info/30eabf6e32ae4b3fbaa7a86bf4796c24 2024-11-20T17:51:08,076 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/info/30eabf6e32ae4b3fbaa7a86bf4796c24, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T17:51:08,077 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/ns/435491b806024e379ac96b4ec30c1312 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/ns/435491b806024e379ac96b4ec30c1312 2024-11-20T17:51:08,080 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/ns/435491b806024e379ac96b4ec30c1312, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T17:51:08,080 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/rep_barrier/4b6e4dbfed774f6d909261a464e4a355 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/rep_barrier/4b6e4dbfed774f6d909261a464e4a355 2024-11-20T17:51:08,083 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/rep_barrier/4b6e4dbfed774f6d909261a464e4a355, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T17:51:08,083 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/.tmp/table/fc441ded672d47cbb410d05dd5413c2f as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/table/fc441ded672d47cbb410d05dd5413c2f 2024-11-20T17:51:08,085 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/table/fc441ded672d47cbb410d05dd5413c2f, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T17:51:08,086 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1731ms, sequenceid=90, compaction requested=false 2024-11-20T17:51:08,089 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T17:51:08,090 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T17:51:08,090 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:51:08,090 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732125066355Running coprocessor pre-close hooks at 1732125066355Disabling compacts and flushes for region at 1732125066355Disabling writes for close at 1732125066355Obtaining lock to block concurrent updates at 1732125066355Preparing flush snapshotting stores in 1588230740 at 1732125066355Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732125066356 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732125066356Flushing 1588230740/info: creating writer at 1732125066356Flushing 1588230740/info: appending metadata at 1732125066378 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732125066378Flushing 1588230740/ns: creating writer at 1732125066797 (+419 ms)Flushing 1588230740/ns: appending metadata at 1732125066810 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732125066810Flushing 1588230740/rep_barrier: creating writer at 1732125067222 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732125067242 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732125067242Flushing 1588230740/table: creating writer at 1732125067650 (+408 ms)Flushing 1588230740/table: appending metadata at 1732125067666 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732125067666Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ab7d7ae: reopening flushed file at 1732125068073 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fc856b7: reopening flushed file at 1732125068076 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13f9831b: reopening flushed file at 1732125068080 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19b22221: reopening flushed file at 1732125068083 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1731ms, sequenceid=90, compaction requested=false at 1732125068086 (+3 ms)Writing region close event to WAL at 1732125068087 (+1 ms)Running coprocessor post-close hooks at 1732125068090 (+3 ms)Closed at 1732125068090 2024-11-20T17:51:08,090 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T17:51:08,157 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,44957,1732124872893; all regions closed. 2024-11-20T17:51:08,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741834_1010 (size=25344) 2024-11-20T17:51:08,163 DEBUG [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/oldWALs 2024-11-20T17:51:08,164 INFO [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C44957%2C1732124872893.meta:.meta(num 1732124876774) 2024-11-20T17:51:08,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741832_1008 (size=15671061) 2024-11-20T17:51:08,166 DEBUG [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/oldWALs 2024-11-20T17:51:08,166 INFO [RS:0;613581fcdc40:44957 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C44957%2C1732124872893:(num 1732124875792) 2024-11-20T17:51:08,166 DEBUG [RS:0;613581fcdc40:44957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T17:51:08,166 INFO [RS:0;613581fcdc40:44957 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T17:51:08,167 INFO [RS:0;613581fcdc40:44957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:51:08,167 INFO [RS:0;613581fcdc40:44957 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T17:51:08,167 INFO [RS:0;613581fcdc40:44957 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:51:08,167 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:51:08,167 INFO [RS:0;613581fcdc40:44957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44957 2024-11-20T17:51:08,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:51:08,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,44957,1732124872893 2024-11-20T17:51:08,236 INFO [RS:0;613581fcdc40:44957 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:51:08,287 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,44957,1732124872893] 2024-11-20T17:51:08,309 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,44957,1732124872893 already deleted, retry=false 2024-11-20T17:51:08,309 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 613581fcdc40,44957,1732124872893 expired; onlineServers=0 2024-11-20T17:51:08,310 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '613581fcdc40,39775,1732124872132' ***** 2024-11-20T17:51:08,310 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T17:51:08,310 INFO [M:0;613581fcdc40:39775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T17:51:08,310 INFO [M:0;613581fcdc40:39775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T17:51:08,310 DEBUG [M:0;613581fcdc40:39775 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T17:51:08,310 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T17:51:08,310 DEBUG [M:0;613581fcdc40:39775 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T17:51:08,310 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124875444 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732124875444,5,FailOnTimeoutGroup] 2024-11-20T17:51:08,310 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124875449 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732124875449,5,FailOnTimeoutGroup] 2024-11-20T17:51:08,311 INFO [M:0;613581fcdc40:39775 {}] hbase.ChoreService(370): Chore service for: master/613581fcdc40:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T17:51:08,311 INFO [M:0;613581fcdc40:39775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T17:51:08,312 DEBUG [M:0;613581fcdc40:39775 {}] master.HMaster(1795): Stopping service threads 2024-11-20T17:51:08,312 INFO [M:0;613581fcdc40:39775 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T17:51:08,312 INFO [M:0;613581fcdc40:39775 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T17:51:08,312 ERROR [M:0;613581fcdc40:39775 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (726517346) connection to localhost/127.0.0.1:38835 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38835,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T17:51:08,314 INFO [M:0;613581fcdc40:39775 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T17:51:08,314 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T17:51:08,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T17:51:08,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:51:08,320 DEBUG [M:0;613581fcdc40:39775 {}] zookeeper.ZKUtil(347): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T17:51:08,320 WARN [M:0;613581fcdc40:39775 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T17:51:08,320 INFO [M:0;613581fcdc40:39775 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/.lastflushedseqids 2024-11-20T17:51:08,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742528_1704 (size=72) 2024-11-20T17:51:08,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:51:08,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44957-0x101595c05f50001, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:51:08,388 INFO [RS:0;613581fcdc40:44957 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:51:08,389 INFO [RS:0;613581fcdc40:44957 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,44957,1732124872893; zookeeper connection closed. 2024-11-20T17:51:08,389 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@fe9dd46 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@fe9dd46 2024-11-20T17:51:08,390 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T17:51:08,731 INFO [M:0;613581fcdc40:39775 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T17:51:08,731 INFO [M:0;613581fcdc40:39775 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T17:51:08,731 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:51:08,731 INFO [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:51:08,731 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:51:08,731 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:51:08,731 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:51:08,731 INFO [M:0;613581fcdc40:39775 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=789.50 KB heapSize=973.24 KB 2024-11-20T17:51:08,745 DEBUG [M:0;613581fcdc40:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bdd8cc2283b417bae6accd9c6b69152 is 82, key is hbase:meta,,1/info:regioninfo/1732124876856/Put/seqid=0 2024-11-20T17:51:08,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742529_1705 (size=5672) 2024-11-20T17:51:09,148 INFO [M:0;613581fcdc40:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2274 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bdd8cc2283b417bae6accd9c6b69152 2024-11-20T17:51:09,171 DEBUG [M:0;613581fcdc40:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/64da554bc103473c99549a64c598a385 is 2275, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1732124910644/Put/seqid=0 2024-11-20T17:51:09,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742530_1706 (size=45170) 2024-11-20T17:51:09,574 INFO [M:0;613581fcdc40:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=788.95 KB at sequenceid=2274 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/64da554bc103473c99549a64c598a385 2024-11-20T17:51:09,578 INFO [M:0;613581fcdc40:39775 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 64da554bc103473c99549a64c598a385 2024-11-20T17:51:09,597 DEBUG [M:0;613581fcdc40:39775 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bf67c5a80ec4c0c9e1f8d23a3c7a00e is 69, key is 613581fcdc40,44957,1732124872893/rs:state/1732124875483/Put/seqid=0 2024-11-20T17:51:09,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073742531_1707 (size=5156) 2024-11-20T17:51:10,000 INFO [M:0;613581fcdc40:39775 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2274 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bf67c5a80ec4c0c9e1f8d23a3c7a00e 2024-11-20T17:51:10,004 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7bdd8cc2283b417bae6accd9c6b69152 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7bdd8cc2283b417bae6accd9c6b69152 2024-11-20T17:51:10,007 INFO [M:0;613581fcdc40:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7bdd8cc2283b417bae6accd9c6b69152, entries=8, sequenceid=2274, filesize=5.5 K 2024-11-20T17:51:10,007 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/64da554bc103473c99549a64c598a385 as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/64da554bc103473c99549a64c598a385 2024-11-20T17:51:10,010 INFO [M:0;613581fcdc40:39775 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 64da554bc103473c99549a64c598a385 2024-11-20T17:51:10,010 INFO [M:0;613581fcdc40:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/64da554bc103473c99549a64c598a385, entries=176, sequenceid=2274, filesize=44.1 K 2024-11-20T17:51:10,010 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bf67c5a80ec4c0c9e1f8d23a3c7a00e as hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bf67c5a80ec4c0c9e1f8d23a3c7a00e 2024-11-20T17:51:10,012 INFO [M:0;613581fcdc40:39775 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bf67c5a80ec4c0c9e1f8d23a3c7a00e, entries=1, sequenceid=2274, filesize=5.0 K 2024-11-20T17:51:10,013 INFO [M:0;613581fcdc40:39775 {}] regionserver.HRegion(3140): Finished flush of dataSize ~789.50 KB/808453, heapSize ~972.95 KB/996296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2274, compaction requested=false 2024-11-20T17:51:10,014 INFO [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:51:10,014 DEBUG [M:0;613581fcdc40:39775 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732125068731Disabling compacts and flushes for region at 1732125068731Disabling writes for close at 1732125068731Obtaining lock to block concurrent updates at 1732125068731Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732125068731Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=808453, getHeapSize=996536, getOffHeapSize=0, getCellsCount=2475 at 1732125068732 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732125068732Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732125068732Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732125068744 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732125068744Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732125069151 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732125069170 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732125069170Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732125069578 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732125069597 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732125069597Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73faa15d: reopening flushed file at 1732125070004 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f948ed: reopening flushed file at 1732125070007 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a0b22d7: reopening flushed file at 1732125070010 (+3 ms)Finished flush of dataSize ~789.50 KB/808453, heapSize ~972.95 KB/996296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2274, compaction requested=false at 1732125070013 (+3 ms)Writing region close event to WAL at 1732125070014 (+1 ms)Closed at 1732125070014 2024-11-20T17:51:10,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36597 is added to blk_1073741830_1006 (size=957486) 2024-11-20T17:51:10,017 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4e3fb769-5ab9-ecbe-c79a-dd5a5c4d65a7/MasterData/WALs/613581fcdc40,39775,1732124872132/613581fcdc40%2C39775%2C1732124872132.1732124874621 not finished, retry = 0 2024-11-20T17:51:10,118 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T17:51:10,118 INFO [M:0;613581fcdc40:39775 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T17:51:10,118 INFO [M:0;613581fcdc40:39775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39775 2024-11-20T17:51:10,118 INFO [M:0;613581fcdc40:39775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T17:51:10,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:51:10,331 INFO [M:0;613581fcdc40:39775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T17:51:10,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39775-0x101595c05f50000, quorum=127.0.0.1:57706, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T17:51:10,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T17:51:10,340 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T17:51:10,340 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T17:51:10,340 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T17:51:10,340 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/hadoop.log.dir/,STOPPED} 2024-11-20T17:51:10,344 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T17:51:10,344 WARN [BP-1760351784-172.17.0.2-1732124868028 heartbeating to localhost/127.0.0.1:38835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T17:51:10,344 WARN [BP-1760351784-172.17.0.2-1732124868028 heartbeating to localhost/127.0.0.1:38835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1760351784-172.17.0.2-1732124868028 (Datanode Uuid 9dae55b9-7ff8-4bc5-9284-abf32dffc5e9) service to localhost/127.0.0.1:38835 2024-11-20T17:51:10,344 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T17:51:10,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/data/data1/current/BP-1760351784-172.17.0.2-1732124868028 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T17:51:10,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/cluster_72ff978c-b7d7-fbaa-fda3-b86bf265295b/data/data2/current/BP-1760351784-172.17.0.2-1732124868028 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T17:51:10,349 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T17:51:10,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T17:51:10,357 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T17:51:10,357 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T17:51:10,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T17:51:10,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d20a683e-cb52-fb49-1b39-76b1128d8249/hadoop.log.dir/,STOPPED} 2024-11-20T17:51:10,372 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T17:51:10,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down